diff --git a/pyproject.toml b/pyproject.toml index 0daf313f..bc7aac7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey" [project.scripts] murfey = "murfey.client:run" "murfey.add_user" = "murfey.cli.add_user:run" +"murfey.build_images" = "murfey.cli.build_images:run" "murfey.create_db" = "murfey.cli.create_db:run" "murfey.db_sql" = "murfey.cli.murfey_db_sql:run" "murfey.decrypt_password" = "murfey.cli.decrypt_db_password:run" diff --git a/src/murfey/cli/build_images.py b/src/murfey/cli/build_images.py new file mode 100644 index 00000000..2232cdea --- /dev/null +++ b/src/murfey/cli/build_images.py @@ -0,0 +1,303 @@ +""" +Helper function to automate the process of building and publishing Docker images for +Murfey using Python subprocesses. + +This CLI is designed to run with Podman commands and in a bash shell that has been +configured to push to a valid Docker repo, which has to be specified using a flag. +""" + +import grp +import os +import subprocess +from argparse import ArgumentParser +from pathlib import Path + + +def run_subprocess(cmd: list[str], src: str = "."): + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + universal_newlines=True, + env=os.environ, + cwd=Path(src), + ) + + # Parse stdout and stderr + if process.stdout: + for line in process.stdout: + print(line, end="") + if process.stderr: + for line in process.stderr: + print(line, end="") + + # Wait for process to complete + process.wait() + + return process.returncode + + +def build_image( + image: str, + tag: str, + source: str, + destination: str, + user_id: int, + group_id: int, + group_name: str, + dry_run: bool = False, +): + # Construct path to Dockerfile + dockerfile = Path(source) / "Dockerfiles" / image + if not dockerfile.exists(): + raise FileNotFoundError( + f"Unable to find Dockerfile for {image} at {str(dockerfile)!r}" + ) + + # Construct tag + image_path = f"{destination}/{image}" + if tag: + image_path = f"{image_path}:{tag}" + + # Construct bash command to build image + build_cmd = [ + "podman build", + f"--build-arg=userid={user_id}", + f"--build-arg=groupid={group_id}", + f"--build-arg=groupname={group_name}", + "--no-cache", + f"-f {str(dockerfile)}", + f"-t {image_path}", + f"{source}", + ] + bash_cmd = ["bash", "-c", " ".join(build_cmd)] + + if not dry_run: + print() + # Run subprocess command to build image + result = run_subprocess(bash_cmd, source) + + # Check for errors + if result != 0: + raise RuntimeError(f"Build command failed with exit code {result}") + + if dry_run: + print() + print(f"Will build image {image!r}") + print(f"Will use Dockerfile from {str(dockerfile)!r}") + print( + f"Will build image with UID {user_id}, GID {group_id}, and group name {group_name}" + ) + print(f"Will build image with tag {image_path}") + print("Will run the following bash command:") + print(bash_cmd) + + return image_path + + +def tag_image( + image_path: str, + tags: list[str], + dry_run: bool = False, +): + # Construct list of tags to create + base_path = image_path.split(":")[0] + new_tags = [f"{base_path}:{tag}" for tag in tags] + + # Construct bash command to add all additional tags + tag_cmd = [ + f"for IMAGE in {' '.join(new_tags)};", + f"do podman tag {image_path} $IMAGE;", + "done", + ] + bash_cmd = ["bash", "-c", " ".join(tag_cmd)] + if not dry_run: + print() + # Run subprocess command to tag images + result = run_subprocess(bash_cmd) + + # Check for errors + if result != 0: + raise RuntimeError(f"Tag command failed with exit code {result}") + + if dry_run: + print() + print("Will run the following bash command:") + print(bash_cmd) + for tag in new_tags: + print(f"Will create new tag {tag}") + + return new_tags + + +def push_images( + images: list[str], + dry_run: bool = False, +): + # Construct bash command to push images to the repo + push_cmd = [f"for IMAGE in {' '.join(images)};", "do podman push $IMAGE;", "done"] + bash_cmd = ["bash", "-c", " ".join(push_cmd)] + if not dry_run: + print() + # Run subprocess command + result = run_subprocess(bash_cmd) + + # Check for errors + if result != 0: + raise RuntimeError(f"Push command failed with exit code {result}") + + if dry_run: + print() + print("Will run the following bash command:") + print(bash_cmd) + for image in images: + print(f"Will push image {image}") + + return True + + +def cleanup(dry_run: bool = False): + # Construct bash command to clean up Podman repo + cleanup_cmd = [ + "podman image prune -f", + ] + bash_cmd = ["bash", "-c", " ".join(cleanup_cmd)] + if not dry_run: + print() + # Run subprocess command + result = run_subprocess(bash_cmd) + + # Check for errors + if result != 0: + raise RuntimeError(f"Cleanup command failed with exit code {result}") + + if dry_run: + print() + print("Will run the following bash command:") + print(bash_cmd) + + return True + + +def run(): + + parser = ArgumentParser( + description=( + "Uses Podman to build, tag, and push the specified images either locally " + "or to a remote repository" + ) + ) + + parser.add_argument( + "images", + nargs="+", + help=("Space-separated list of Murfey Dockerfiles that you want to build."), + ) + + parser.add_argument( + "--tags", + "-t", + nargs="*", + default=["latest"], + help=("Space-separated list of tags to apply to the built images"), + ) + + parser.add_argument( + "--source", + "-s", + default=".", + help=("Directory path to the Murfey repository"), + ) + + parser.add_argument( + "--destination", + "-d", + default="localhost", + help=("The URL of the repo to push the built images to"), + ) + + parser.add_argument( + "--user-id", + default=os.getuid(), + help=("The user ID to install in the images"), + ) + + parser.add_argument( + "--group-id", + default=os.getgid(), + help=("The group ID to install in the images"), + ) + + parser.add_argument( + "--group-name", + default=( + grp.getgrgid(os.getgid()).gr_name if hasattr(grp, "getgrgid") else "nogroup" + ), + help=("The group name to install in the images"), + ) + + parser.add_argument( + "--dry-run", + default=False, + action="store_true", + help=( + "When specified, prints out what the command would have done for each " + "stage of the process" + ), + ) + + args = parser.parse_args() + + # Validate the paths to the images + for image in args.images: + if not (Path(args.source) / "Dockerfiles" / str(image)).exists(): + raise FileNotFoundError( + "No Dockerfile found in " + f"source repository {str(Path(args.source).resolve())!r} for" + f"image {str(image)!r}" + ) + + # Build image + images = [] + for image in args.images: + image_path = build_image( + image=image, + tag=args.tags[0], + source=args.source, + destination=( + str(args.destination).rstrip("/") + if str(args.destination).endswith("/") + else str(args.destination) + ), + user_id=args.user_id, + group_id=args.group_id, + group_name=args.group_name, + dry_run=args.dry_run, + ) + images.append(image_path) + + # Create additional tags (if any) for each built image + if len(args.tags) > 1: + new_tags = tag_image( + image_path=image_path, + tags=args.tags[1:], + dry_run=args.dry_run, + ) + images.extend(new_tags) + + # Push all built images to specified repo + push_images(images=images, dry_run=args.dry_run) + + # Perform final cleanup + cleanup(dry_run=args.dry_run) + + # Notify that job is completed + print() + print("Done") + + +# Allow it to be run directly from the file +if __name__ == "__main__": + run() diff --git a/tests/cli/test_build_images.py b/tests/cli/test_build_images.py new file mode 100644 index 00000000..86489b11 --- /dev/null +++ b/tests/cli/test_build_images.py @@ -0,0 +1,338 @@ +import grp +import os +import sys +from unittest.mock import call, patch + +import pytest + +from murfey.cli.build_images import build_image, cleanup, push_images, run, tag_image + +images = [f"test_image_{n}" for n in range(3)] + +# Set defaults of the various flags +def_tags = ["latest"] +def_src = "." +def_dst = "localhost" +def_uid = os.getuid() +def_gid = os.getgid() +def_gname = grp.getgrgid(os.getgid()).gr_name if hasattr(grp, "getgrgid") else "nogroup" +def_dry_run = False + + +build_image_params_matrix: tuple[ + tuple[list[str], list[str], str, str, int, int, str, bool], ... +] = ( + # Images | Tags | Source | Destination | User ID | Group ID | Group Name | Dry Run + # Populated flags + ( + images, + ["latest", "dev", "1.1.1"], + "", + "docker.io", + 12345, + 34567, + "my-group", + False, + ), + ( + images, + ["latest", "dev", "1.1.1"], + "", + "docker.io", + 12345, + 34567, + "my-group", + True, + ), +) + + +@pytest.mark.parametrize("build_params", build_image_params_matrix) +@patch("murfey.cli.build_images.Path.exists") +@patch("murfey.cli.build_images.run_subprocess") +def test_build_image(mock_subprocess, mock_exists, build_params): + + # Unpack build params + images, tags, src, dst, uid, gid, gname, dry_run = build_params + + # Set the return values for 'Path().exists()' and 'run_subprocess' + mock_exists.return_value = True + mock_subprocess.return_value = 0 + + # Run the command + built_image = build_image( + image=images[0], + tag=tags[0], + source=src, + destination=dst, + user_id=uid, + group_id=gid, + group_name=gname, + dry_run=dry_run, + ) + + # Check that the image path generated is correct + assert built_image == f"{dst}/{images[0]}:{tags[0]}" + + +tag_image_params_matrix: tuple[tuple[list[str], list[str], str, bool], ...] = ( + # Images | Tags | Destination | Dry Run + # Populated flags + ( + images, + ["latest", "dev", "1.1.1"], + "docker.io", + False, + ), + ( + images, + ["latest", "dev", "1.1.1"], + "docker.io", + True, + ), +) + + +@pytest.mark.parametrize("tag_params", tag_image_params_matrix) +@patch("murfey.cli.build_images.run_subprocess") +def test_tag_image(mock_subprocess, tag_params): + + # Unpack build params + images, tags, dst, dry_run = tag_params + + # Check that the image path generated is correct + built_image = f"{dst}/{images[0]}:{tags[0]}" + + # Set the return value for 'run_subprocess' + mock_subprocess.return_value = 0 + + # Run the command + image_tags = tag_image( + image_path=built_image, + tags=tags[1:], + dry_run=dry_run, + ) + + # Check that the images are tagged correctly + assert image_tags == [f"{built_image.split(':')[0]}:{tag}" for tag in tags[1:]] + + +push_image_params_matrix: tuple[tuple[list[str], list[str], str, bool], ...] = ( + # Images | Tags | Destination | Dry Run + # Populated flags + ( + images, + ["latest", "dev", "1.1.1"], + "docker.io", + False, + ), + ( + images, + ["latest", "dev", "1.1.1"], + "docker.io", + True, + ), +) + + +@pytest.mark.parametrize("push_params", push_image_params_matrix) +@patch("murfey.cli.build_images.run_subprocess") +def test_push_images( + mock_subprocess, + push_params, +): + + # Unpack test parameters + images, tags, dst, dry_run = push_params + + # Construct all images to be pushed + images_to_push = [f"{dst}/{image}:{tag}" for image in images for tag in tags] + + # Mock the subprocess return value + mock_subprocess.return_value = 0 + + # Run the function + result = push_images( + images=images_to_push, + dry_run=dry_run, + ) + assert result + + +test_cleanup_params_matrix: tuple[tuple[bool], ...] = ((True,), (False,)) + + +@pytest.mark.parametrize("cleanup_params", test_cleanup_params_matrix) +@patch("murfey.cli.build_images.run_subprocess") +def test_cleanup( + mock_subprocess, + cleanup_params, +): + + # Unpack test params + (dry_run,) = cleanup_params + + # Mock the subprocess return value + mock_subprocess.return_value = 0 + + # Run the function + result = cleanup(dry_run) + assert result + + +test_run_params_matrix: tuple[ + tuple[list[str], list[str], str, str, str, str, str, bool], ... +] = ( + # Images | Tags | Source | Destination | User ID | Group ID | Group Name | Dry Run + # Default settings + (images, [], "", "", "", "", "", False), + # Populated flags + ( + images, + ["latest", "dev", "1.1.1"], + "", + "docker.io", + "12345", + "34567", + "my-group", + False, + ), + ( + images, + ["latest", "dev", "1.1.1"], + "", + "docker.io", + "12345", + "34567", + "my-group", + True, + ), +) + + +@pytest.mark.parametrize("run_params", test_run_params_matrix) +@patch("murfey.cli.build_images.Path.exists") +@patch("murfey.cli.build_images.run_subprocess") +@patch("murfey.cli.build_images.cleanup") +@patch("murfey.cli.build_images.push_images") +@patch("murfey.cli.build_images.tag_image") +@patch("murfey.cli.build_images.build_image") +def test_run( + mock_build, + mock_tag, + mock_push, + mock_clean, + mock_subprocess, + mock_exists, + run_params: tuple[list[str], list[str], str, str, str, str, str, bool], +): + """ + Tests that the function is run with the expected arguments for a given + combination of flags. + """ + + # Unpack build params + images, tags, src, dst, uid, gid, gname, dry_run = run_params + + # Set up the command based on what these values are + build_cmd = [ + "murfey.build_images", + *images, + ] + + # Iterate through flags and add them according to the command + flags = ( + # 'images' already include by default + ("--tags", tags), + ("--source", src), + ("--destination", dst), + ("--user-id", uid), + ("--group-id", gid), + ("--group-name", gname), + ("--dry-run", dry_run), + ) + for flag, value in flags: + if isinstance(value, list) and value: + build_cmd.extend([flag, *value]) + if isinstance(value, str) and value: + build_cmd.extend([flag, value]) + if isinstance(value, bool) and value: + build_cmd.append(flag) + + # Assign it to the CLI to pass to the function + sys.argv = build_cmd + + # Mock the check for the existence of the Dockerfiles + mock_exists.return_value = True + + # Mock the exit code of the subprocesses being run + mock_subprocess.return_value = 0 + + # Construct images that will be generated at the different stages of the process + built_images: list[str] = [] + other_tags: list[list[str]] = [] + images_to_push: list[str] = [] + for image in images: + built_image = ( + f"{dst if dst else def_dst}/{image}:{tags[0] if tags else def_tags[0]}" + ) + built_images.append(built_image) + images_to_push.append(built_image) + new_tags = [ + f"{built_image.split(':')[0]}:{tag}" + for tag in (tags if tags else def_tags)[1:] + ] + other_tags.append(new_tags) + images_to_push.extend(new_tags) + + # Mock the return values of 'build_image' and 'tag_iamge' + mock_build.side_effect = built_images + mock_tag.side_effect = other_tags + + # Mock the push and cleanup functions + mock_push.return_value = True + mock_clean.return_value = True + + # Run the function with the command + run() + + # Check that 'build_image' was called with the correct arguments + assert mock_build.call_count == len(images) + expected_build_calls = ( + call( + image=image, + tag=tags[0] if tags else def_tags[0], + source=src if src else def_src, + destination=dst if dst else def_dst, + user_id=uid if uid else def_uid, + group_id=gid if gid else def_gid, + group_name=gname if gname else def_gname, + dry_run=dry_run if dry_run else def_dry_run, + ) + for image in images + ) + mock_build.assert_has_calls(expected_build_calls, any_order=True) + + # Check that 'tag_image' was called with the correct arguments + if tags[1:]: + assert mock_tag.call_count == len(built_images) + expected_tag_calls = ( + call( + image_path=image, + tags=tags[1:], + dry_run=dry_run if dry_run else def_dry_run, + ) + for image in built_images + ) + mock_tag.assert_has_calls(expected_tag_calls, any_order=True) + + # Check that 'push_images' was called with the correct arguments + mock_push.assert_called_once_with( + images=images_to_push, + dry_run=dry_run if dry_run else def_dry_run, + ) + + # Check that 'cleanup' was called correctly + mock_clean.assert_called_once_with( + dry_run=dry_run if dry_run else def_dry_run, + )