diff --git a/faasmctl/tasks/__init__.py b/faasmctl/tasks/__init__.py index c587908..12c28c9 100644 --- a/faasmctl/tasks/__init__.py +++ b/faasmctl/tasks/__init__.py @@ -9,6 +9,7 @@ from . import logs from . import monitor from . import restart +from . import s3 from . import scale from . import status from . import upload @@ -23,6 +24,7 @@ logs, monitor, restart, + s3, scale, status, upload, diff --git a/faasmctl/tasks/s3.py b/faasmctl/tasks/s3.py new file mode 100644 index 0000000..8563bc5 --- /dev/null +++ b/faasmctl/tasks/s3.py @@ -0,0 +1,59 @@ +from invoke import task +from faasmctl.util.s3 import ( + list_buckets as do_list_buckets, + clear_bucket as do_clear_bucket, + list_objects as do_list_objects, + upload_file as do_upload_file, + upload_dir as do_upload_dir, + dump_object as do_dump_object, +) + + +@task +def list_buckets(ctx): + """ + List available buckets + """ + do_list_buckets() + + +@task +def clear_bucket(ctx, bucket): + """ + Clear (i.e. remove) bucket + """ + do_clear_bucket(bucket) + + +@task +def list_objects(ctx, bucket, recursive=False): + """ + List available objects in bucket + """ + do_list_objects(bucket, recursive) + + +@task +def upload_file(ctx, bucket, host_path, s3_path): + """ + Upload a file to S3 + """ + do_upload_file(bucket, host_path, s3_path) + + +@task +def upload_dir(ctx, bucket, host_path, s3_path): + """ + Upload all the files in a directory to S3 + """ + do_upload_dir(bucket, host_path, s3_path) + + +@task +def dump_object(ctx, bucket, path): + """ + Dump the contents of an object in S3 + """ + response = do_dump_object(bucket, path) + + print(response.data) diff --git a/faasmctl/util/s3.py b/faasmctl/util/s3.py new file mode 100644 index 0000000..3949ad5 --- /dev/null +++ b/faasmctl/util/s3.py @@ -0,0 +1,73 @@ +from faasmctl.util.config import get_faasm_ini_file, get_faasm_ini_value +from minio import Minio +from minio.error import S3Error +from os import listdir +from os.path import isfile, join + + +def get_minio_client(): + minio_port = get_faasm_ini_value(get_faasm_ini_file(), "Faasm", "minio_port") + + client = Minio( + "localhost:{}".format(minio_port), + access_key="minio", + secret_key="minio123", + secure=False, + region="", + ) + + return client + + +def list_buckets(): + client = get_minio_client() + for bucket in client.list_buckets(): + print(bucket) + + +def list_objects(bucket, recursive=False): + client = get_minio_client() + for bucket_key in client.list_objects(bucket, recursive=recursive): + print(bucket_key.object_name) + + +def clear_bucket(bucket): + client = get_minio_client() + + # Got to make sure the bucket is empty first + for bucket_key in client.list_objects(bucket, recursive=True): + client.remove_object(bucket, bucket_key.object_name) + + client.remove_bucket(bucket) + + +def upload_file(bucket, host_path, s3_path): + client = get_minio_client() + + # Create the bucket if it does not exist + found = client.bucket_exists(bucket) + if not found: + client.make_bucket(bucket) + + # Upload the file, renaming it in the process + try: + client.fput_object(bucket, s3_path, host_path) + except S3Error as ex: + print("error: error uploading file to s3: {}".format(ex)) + raise RuntimeError("error: error uploading file to s3") + + +def upload_dir(bucket, host_path, s3_path): + for f in listdir(host_path): + host_file_path = join(host_path, f) + + if isfile(host_file_path): + s3_file_path = join(s3_path, f) + upload_file(bucket, host_file_path, s3_file_path) + + +def dump_object(bucket, path): + client = get_minio_client() + response = client.get_object(bucket, path) + + return response diff --git a/pyproject.toml b/pyproject.toml index 754353a..179d77d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,10 @@ classifiers = [ "Operating System :: OS Independent", ] dependencies = [ - "invoke == 2.1.3", - "requests == 2.31.0", - "protobuf == 4.23.4", + "invoke >= 2.1.3", + "requests >= 2.31.0", + "protobuf >= 4.23.4", + "minio >= 7.2.7", ] [project.urls]