Import precheck: Support disk checking back to EL7 (#1743)

Prior to this PR, the system's block device hierarchy was fetched using lsblk --json. This caused errors in systems that did not support this flag for lsblk, eg #1736.

cli_tools/common/mount/inspector.go

@@ -0,0 +1,121 @@
+//  Copyright 2021 Google Inc. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package mount
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/shell"
+)
+
+// To rebuild mocks, run `go generate ./...`
+//go:generate go run github.com/golang/mock/mockgen -package mount -source $GOFILE -destination mock_mount_inspector.go
+
+// Inspector inspects a mount directory to return:
+//   - The underlying block device.
+//   - The block device's type.
+//   - If the block device is virtual, the number of block devices
+//     that comprise it.
+type Inspector interface {
+	Inspect(dir string) (InspectionResults, error)
+}
+
+// InspectionResults contains information about the mountpoint of a directory.
+type InspectionResults struct {
+	BlockDevicePath        string
+	BlockDeviceIsVirtual   bool
+	UnderlyingBlockDevices []string
+}
+
+// NewMountInspector returns a new inspector that uses command-line utilities.
+func NewMountInspector() Inspector {
+	return &defaultMountInspector{shell.NewShellExecutor()}
+}
+
+type defaultMountInspector struct {
+	shellExecutor shell.Executor
+}
+
+// Inspect returns the mount information for dir.
+func (mi *defaultMountInspector) Inspect(dir string) (mountInfo InspectionResults, err error) {
+	if mountInfo.BlockDevicePath, err = mi.getDeviceForMount(dir); err != nil {
+		return InspectionResults{}, fmt.Errorf("unable to find mount information for `%s`: %w", dir, err)
+	}
+	if mountInfo.BlockDeviceIsVirtual, err = mi.isDeviceVirtual(mountInfo.BlockDevicePath); err != nil {
+		return InspectionResults{}, fmt.Errorf("unable to find the type of device `%s`: %w", mountInfo.BlockDevicePath, err)
+	}
+	if mountInfo.UnderlyingBlockDevices, err = mi.getPhysicalDisks(mountInfo.BlockDevicePath); err != nil {
+		return InspectionResults{}, fmt.Errorf("unable to find the physical disks for the block device `%s`: %w",
+			mountInfo.BlockDevicePath, err)
+	}
+	return mountInfo, nil
+}
+
+// getDeviceForMount returns the path of the block device that is mounted for dir.
+func (mi *defaultMountInspector) getDeviceForMount(dir string) (string, error) {
+	stdout, err := mi.shellExecutor.Exec("getDeviceForMount", "--noheadings", "--output=SOURCE", dir)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(stdout), nil
+}
+
+// isDeviceVirtual returns whether the block device is LVM
+func (mi *defaultMountInspector) isDeviceVirtual(device string) (bool, error) {
+	stdout, err := mi.shellExecutor.Exec("lsblk", "--noheadings", "--output=TYPE", device)
+	return strings.TrimSpace(strings.ToLower(stdout)) == "lvm", err
+}
+
+// getPhysicalDisks returns the list of physical disks that are used for the blockDevice.
+// For example:
+//   1. blockDevice is an MBR-style partition:
+//         blockDevice: /dev/sdb1
+//         getPhysicalDisks:  [/dev/sdb]
+//   2. blockDevice is an LVM logical volume that is spread across three disks:
+//         blockDevice:  /dev/mapper/vg-lv
+//         getPhysicalDisks:  [/dev/sda, /dev/sdb, /dev/sdc]
+func (mi *defaultMountInspector) getPhysicalDisks(blockDevice string) (disksForDevice []string, err error) {
+	disks, err := mi.getAllPhysicalDisks()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, disk := range disks {
+		blkDevices, err := mi.blockDevicesOnDisk(disk)
+		if err != nil {
+			return nil, err
+		}
+		for _, blkDevice := range blkDevices {
+			if blkDevice == blockDevice {
+				disksForDevice = append(disksForDevice, disk)
+				break
+			}
+		}
+	}
+	return disksForDevice, nil
+}
+
+// getAllPhysicalDisks returns the paths of all physical disks on the system.
+func (mi *defaultMountInspector) getAllPhysicalDisks() (allDisks []string, err error) {
+	return mi.shellExecutor.ExecLines(
+		"lsblk", "--noheadings", "--paths", "--list", "--nodeps", "--output=NAME")
+}
+
+// blockDevicesOnDisk returns the paths of all block devices contained on a disk.
+func (mi *defaultMountInspector) blockDevicesOnDisk(disk string) ([]string, error) {
+	return mi.shellExecutor.ExecLines(
+		"lsblk", "--noheadings", "--paths", "--list", "--output=NAME", disk)
+}

cli_tools/common/mount/inspector_test.go

@@ -0,0 +1,143 @@
+//  Copyright 2021 Google Inc. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package mount
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/mocks"
+)
+
+func TestMountInspector_Inspect_HappyCase_NotVirtual(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	mockShell := mocks.NewMockShellExecutor(mockCtrl)
+	setupRootMount(mockShell, "/dev/sdb1", "part")
+	setupPhysicalDisks(mockShell, map[string][]string{
+		"/dev/sda": {"/dev/sda", "/dev/sda1"},
+		"/dev/sdb": {"/dev/sdb", "/dev/sdb1"},
+	})
+
+	mountInspector := &defaultMountInspector{mockShell}
+	mountInfo, err := mountInspector.Inspect("/")
+	assert.NoError(t, err)
+	assert.Equal(t, InspectionResults{
+		BlockDevicePath:        "/dev/sdb1",
+		BlockDeviceIsVirtual:   false,
+		UnderlyingBlockDevices: []string{"/dev/sdb"},
+	}, mountInfo)
+}
+
+func TestMountInspector_Inspect_HappyCase_Virtual(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	mockShell := mocks.NewMockShellExecutor(mockCtrl)
+	setupRootMount(mockShell, "/dev/mapper/vg-device", "lvm")
+	setupPhysicalDisks(mockShell, map[string][]string{
+		"/dev/sda": {"/dev/sda", "/dev/sda1", "/dev/mapper/vg-device"},
+		"/dev/sdb": {"/dev/sdb", "/dev/sdb1", "/dev/mapper/vg-device"},
+	})
+
+	mountInspector := &defaultMountInspector{mockShell}
+	mountInfo, err := mountInspector.Inspect("/")
+	assert.NoError(t, err)
+	assert.Equal(t, InspectionResults{
+		BlockDevicePath:        "/dev/mapper/vg-device",
+		BlockDeviceIsVirtual:   true,
+		UnderlyingBlockDevices: []string{"/dev/sda", "/dev/sdb"},
+	}, mountInfo)
+}
+
+func TestMountInspector_Inspect_PropagatesErrorFromFindMnt(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	mockShell := mocks.NewMockShellExecutor(mockCtrl)
+	mockShell.EXPECT().Exec("getDeviceForMount", "--noheadings",
+		"--output=SOURCE", "/").Return("", errors.New("[getDeviceForMount] not executable"))
+
+	mountInspector := &defaultMountInspector{mockShell}
+	_, err := mountInspector.Inspect("/")
+	assert.Equal(t, err.Error(), "unable to find mount information for `/`: [getDeviceForMount] not executable")
+}
+
+func TestMountInspector_Inspect_PropagatesErrorFromGettingDeviceType(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	mockShell := mocks.NewMockShellExecutor(mockCtrl)
+	mockShell.EXPECT().Exec("getDeviceForMount", "--noheadings",
+		"--output=SOURCE", "/").Return("/dev/mapper/vg-device", nil)
+	mockShell.EXPECT().Exec("lsblk", "--noheadings",
+		"--output=TYPE", "/dev/mapper/vg-device").Return("", errors.New("[lsblk] not executable"))
+
+	mountInspector := &defaultMountInspector{mockShell}
+	_, err := mountInspector.Inspect("/")
+	assert.Equal(t, err.Error(), "unable to find the type of device `/dev/mapper/vg-device`: [lsblk] not executable")
+}
+
+func TestMountInspector_Inspect_PropagatesErrorFromGettingAllDisks(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	mockShell := mocks.NewMockShellExecutor(mockCtrl)
+	setupRootMount(mockShell, "/dev/sdb1", "part")
+	mockShell.EXPECT().ExecLines("lsblk", "--noheadings", "--paths", "--list", "--nodeps", "--output=NAME").Return(
+		nil, errors.New("[lsblk] not executable"))
+
+	mountInspector := &defaultMountInspector{mockShell}
+	_, err := mountInspector.Inspect("/")
+	assert.Equal(t, err.Error(), "unable to find the physical disks for the block device `/dev/sdb1`: [lsblk] not executable")
+}
+
+func TestMountInspector_Inspect_PropagatesErrorFromGettingDevicesOnDisk(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	mockShell := mocks.NewMockShellExecutor(mockCtrl)
+	setupRootMount(mockShell, "/dev/sdb1", "part")
+	mockShell.EXPECT().ExecLines("lsblk", "--noheadings", "--paths", "--list", "--nodeps", "--output=NAME").Return(
+		[]string{"/dev/sda", "/dev/sdb"}, nil)
+	mockShell.EXPECT().ExecLines("lsblk", "--noheadings", "--paths", "--list", "--output=NAME", "/dev/sda").Return(
+		nil, errors.New("[lsblk] not executable"))
+
+	mountInspector := &defaultMountInspector{mockShell}
+	_, err := mountInspector.Inspect("/")
+	assert.Equal(t, err.Error(), "unable to find the physical disks for the block device `/dev/sdb1`: [lsblk] not executable")
+}
+
+func setupPhysicalDisks(mockShell *mocks.MockShellExecutor, deviceMap map[string][]string) {
+	var disks []string
+	for disk := range deviceMap {
+		disks = append(disks, disk)
+	}
+	mockShell.EXPECT().ExecLines("lsblk", "--noheadings", "--paths", "--list", "--nodeps", "--output=NAME").Return(
+		disks, nil)
+	for disk, devices := range deviceMap {
+		mockShell.EXPECT().ExecLines("lsblk", "--noheadings", "--paths", "--list", "--output=NAME", disk).Return(
+			devices, nil)
+	}
+}
+
+func setupRootMount(mockShell *mocks.MockShellExecutor, mointPoint string, mointPointType string) {
+	mockShell.EXPECT().Exec("getDeviceForMount", "--noheadings", "--output=SOURCE", "/").Return(mointPoint, nil)
+	mockShell.EXPECT().Exec("lsblk", "--noheadings", "--output=TYPE", mointPoint).Return(mointPointType, nil)
+}

cli_tools/common/utils/shell/executor.go

@@ -0,0 +1,61 @@
+//  Copyright 2021 Google Inc. All Rights Reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package shell
+
+import (
+	"bufio"
+	"bytes"
+	"os/exec"
+)
+
+// To rebuild mocks, run `go generate ./...`
+//go:generate go run github.com/golang/mock/mockgen -package mocks -source $GOFILE -mock_names=Executor=MockShellExecutor -destination ../../../mocks/mock_shell_exececutor.go
+
+// Executor is a shim over cmd.Output() that allows for testing.
+type Executor interface {
+	// Exec executes program with args, and returns stdout if the return code is zero.
+	// If nonzero, stderr is included in error.
+	Exec(program string, args ...string) (string, error)
+	// ExecLines is similar to Exec, except it splits the output on newlines. All empty
+	// lines are discarded.
+	ExecLines(program string, args ...string) ([]string, error)
+}
+
+// NewShellExecutor creates a shell.Executor that is implemented by exec.Command.
+func NewShellExecutor() Executor {
+	return &defaultShellExecutor{}
+}
+
+type defaultShellExecutor struct {
+}
+
+func (d *defaultShellExecutor) Exec(program string, args ...string) (string, error) {
+	cmd := exec.Command(program, args...)
+	stdout, err := cmd.Output()
+	return string(stdout), err
+}
+
+func (d *defaultShellExecutor) ExecLines(program string, args ...string) (allLines []string, err error) {
+	cmd := exec.Command(program, args...)
+	stdout, err := cmd.Output()
+	scanner := bufio.NewScanner(bytes.NewReader(stdout))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line != "" {
+			allLines = append(allLines, line)
+		}
+	}
+	return allLines, err
+}