Obtaining a first version of a working and simple uploader.

Signed-off-by: Cédric Foellmi <[email protected]>

Author: onekiloparsec

oort/cli/cli.py

@@ -1,13 +1,14 @@
 import click
-from arcsecond import ArcsecondAPI, Config
+from arcsecond import ArcsecondAPI, ArcsecondConfig, cli as ArcsecondCLI
 from arcsecond.options import State
 
 from oort import __version__
+from oort.common.context import Context
 from oort.common.utils import build_endpoint_kwargs
+from oort.uploader.walker import walk
 from .errors import OortCloudError, InvalidUploadOptionsOortCloudError
 from .helpers import display_command_summary
 from .options import basic_options
-from .validators import validate_upload_parameters
 
 pass_state = click.make_pass_decorator(State, ensure=True)
 
@@ -64,35 +65,21 @@ def login(state, username, password):
     This Upload key is not your full API key. When logging in with oort, no fetch
     nor storage of the API key occur (only the Upload one).
     """
-    _, error = ArcsecondAPI(Config(state)).login(username, password, upload_key=True)
+    config = ArcsecondConfig(state)
+    _, error = ArcsecondAPI(config).login(username, password, upload_key=True)
     if error:
         click.echo(error)
     else:
-        username = ArcsecondAPI.username(api=state.api_name)
+        username = config.username
         click.echo(f' • Successfully logged in as @{username} (API: {state.api_name}).')
 
 
-@main.command()
-@click.argument('name', required=True, nargs=1)
-@click.argument('address', required=False, nargs=1)
+@main.command(help='Get or set the API server address (fully qualified domain name).')
+@click.argument('name', required=False, nargs=1)
+@click.argument('fqdn', required=False, nargs=1)
 @pass_state
-def api(state, name=None, address=None):
-    """
-    Configure the API server address.
-
-    For instance:
-
-    • "oort api main" to get the main API server address (default).\n
-    • "oort api dev http://localhost:8000" to configure a dev server.
-
-    You can then use --api <api name> in every command to choose which API
-    server you want to interact with. Hence, "--api dev" will choose the above
-    dev server.
-    """
-    if address is None:
-        print(ArcsecondAPI.get_api_name(api_name=name))
-    else:
-        ArcsecondAPI.set_api_name(address, api_name=name)
+def api(state, name=None, fqdn=None):
+    ArcsecondCLI.api(state, name, fqdn)
 
 
 @main.command(help="Display the list of (organisation) datasets.")
@@ -109,7 +96,7 @@ def datasets(state, organisation=None):
         click.echo(" • Fetching datasets...")
 
     kwargs = build_endpoint_kwargs(state.api_name, subdomain=organisation)
-    dataset_list, error = ArcsecondAPI.datasets(**kwargs).list()
+    dataset_list, error = ArcsecondAPI(ArcsecondConfig(state)).datasets.list()
     if error is not None:
         raise OortCloudError(str(error))
 
@@ -134,7 +121,7 @@ def datasets(state, organisation=None):
               help="The subdomain, if uploading for an Observatory Portal.")
 @basic_options
 @pass_state
-def upload(state, folder, organisation=None, dataset=None):
+def upload(state, folder, dataset=None, organisation=None):
     """
     Upload the content of a folder.
 
@@ -152,16 +139,17 @@ def upload(state, folder, organisation=None, dataset=None):
     Oort will then start walking through the folder tree and uploads regular files
     (hidden and empty files will be skipped).
     """
+    config = ArcsecondConfig(state)
+    context = Context(config, dataset_uuid_or_name=dataset, subdomain=organisation)
+
     try:
-        values = validate_upload_parameters(organisation, dataset, state)
+        context.validate()
     except InvalidUploadOptionsOortCloudError as e:
         click.echo(f"\n • ERROR {str(e)} \n")
         return
 
-    display_command_summary([folder, ], Config(state), values)
+    display_command_summary(context, [folder, ])
     ok = input('\n   ----> OK? (Press Enter) ')
 
     if ok.strip() == '':
-        from oort.uploader.walker import walk
-
-        walk(folder, context)
+        walk(context, folder)

oort/cli/helpers.py

@@ -2,8 +2,8 @@
 import pathlib
 
 import click
-from arcsecond import Config
 
+from oort.common.context import Context
 from oort.common.utils import is_file_hidden
 
 
@@ -35,27 +35,25 @@ def __get_formatted_bytes_size(size):
     return f"{(size / math.pow(k, i)):.2f} {units[i]}"
 
 
-def display_command_summary(folders: list, config: Config, values: dict):
+def display_command_summary(context: Context, folders: list):
     click.echo("\n --- Upload summary --- ")
-    click.echo(f" • Arcsecond username: @{config.username} (Upload key: {config.upload_key[:4]}••••)")
-    if config.subdomain:
-        msg = f" • Uploading to Observatory Portal '{config.subdomain}' (as {config.read_key(config.subdomain)})."
+    click.echo(f" • Arcsecond username: @{context.config.username} (Upload key: {context.config.upload_key[:4]}••••)")
+    if context.config.subdomain:
+        role = context.config.read_key(context.config.subdomain)
+        msg = f" • Uploading to Observatory Portal '{context.config.subdomain}' (as {role})."
     else:
         msg = " • Uploading to your *personal* account."
     click.echo(msg)
 
-    dataset_uuid = values.get('dataset').get('uuid', '')
-    dataset_name = values.get('dataset').get('name', '')
-
-    if dataset_uuid and dataset_name:
-        msg = f" • Data will be appended to existing dataset '{dataset_name}' ({dataset_uuid})."
-    elif not dataset_uuid and dataset_name:
-        msg = f" • Data will be inserted into a new dataset named '{dataset_name}'."
+    if context.dataset_uuid and context.dataset_name:
+        msg = f" • Data will be appended to existing dataset '{context.dataset_name}' ({context.dataset_uuid})."
+    elif not context.dataset_uuid and context.dataset_name:
+        msg = f" • Data will be inserted into a new dataset named '{context.dataset_name}'."
     else:
         msg = " • Using folder names for dataset names (one folder = one dataset)."
     click.echo(msg)
 
-    click.echo(f" • Using API server: {config.api_name}")
+    click.echo(f" • Using API server: {context.config.api_name}")
     click.echo(f" • Folder{'s' if len(folders) > 1 else ''}:")
     for folder in folders:
         folder_path = pathlib.Path(folder).expanduser().resolve()

oort/cli/validators.py

@@ -0,0 +1,103 @@
+import uuid
+
+import click
+from arcsecond import ArcsecondAPI, ArcsecondConfig
+
+from oort.cli.errors import (
+    UnknownOrganisationOortCloudError,
+    InvalidOrgMembershipOortCloudError,
+    InvalidAstronomerOortCloudError,
+    InvalidWatchOptionsOortCloudError,
+    InvalidOrganisationDatasetOortCloudError,
+    InvalidDatasetOortCloudError
+)
+
+
+class Context(object):
+    def __init__(self, config: ArcsecondConfig, dataset_uuid_or_name: str, subdomain: str):
+        self._config = config
+        self._dataset_uuid_or_name = dataset_uuid_or_name
+        self._subdomain = subdomain
+        self._dataset = None
+        self._organisation = None
+        self._api = ArcsecondAPI(config, subdomain)
+
+    def validate(self):
+        self._validate_local_astronomer_credentials()
+        self._validate_dataset_uuid()
+        if self._subdomain:
+            self._validate_remote_organisation()
+            self._validate_astronomer_role_in_remote_organisation()
+
+    def _validate_local_astronomer_credentials(self):
+        username = self._config.username
+        if username is None:
+            raise InvalidAstronomerOortCloudError('Missing username')
+
+        upload_key = self._config.upload_key
+        if not upload_key:
+            raise InvalidWatchOptionsOortCloudError('Missing upload_key.')
+
+    def _validate_dataset_uuid(self):
+        try:
+            uuid.UUID(self._dataset_uuid_or_name)
+        except ValueError:
+            click.echo(f" • Looking for a dataset with name {self._dataset_uuid_or_name}...")
+            datasets_list, error = self._api.datasets.list(**{'name': self._dataset_uuid_or_name})
+            if len(datasets_list) == 0:
+                click.echo(f" • No dataset with name {self._dataset_uuid_or_name} found. It will be created.")
+                self._dataset = {'name': self._dataset_uuid_or_name}
+            elif len(datasets_list) == 1:
+                click.echo(f" • One dataset with name {self._dataset_uuid_or_name}. Data will be appended to it.")
+                self._dataset = datasets_list[0]
+            else:
+                error = f"Multiple datasets with name {self._dataset_uuid_or_name} found. Be more specific."
+        else:
+            click.echo(f" • Fetching details of dataset {self._dataset_uuid_or_name}...")
+            self._dataset, error = self._api.datasets.read(str(self._dataset_uuid_or_name))
+
+        if error is not None:
+            if self._subdomain:
+                raise InvalidOrganisationDatasetOortCloudError(str(self._dataset_uuid_or_name),
+                                                               self._subdomain,
+                                                               str(error))
+            else:
+                raise InvalidDatasetOortCloudError(str(self._dataset_uuid_or_name), str(error))
+
+    def _validate_remote_organisation(self):
+        click.echo(f" • Fetching details of organisation {self._subdomain}...")
+        self._organisation, error = self._api.organisations.read(self._subdomain)
+        if error is not None:
+            raise UnknownOrganisationOortCloudError(self._subdomain, str(error))
+
+    def _validate_astronomer_role_in_remote_organisation(self):
+        role = self._config.read_key(self._subdomain)
+        if not role or role not in ['member', 'admin', 'superadmin']:
+            raise InvalidOrgMembershipOortCloudError(self._subdomain)
+
+    @property
+    def config(self):
+        return self._config
+
+    @property
+    def dataset_uuid(self):
+        return self._dataset.get('uuid', '')
+
+    @property
+    def dataset_name(self):
+        return self._dataset.get('name', '')
+
+    @property
+    def organisation_subdomain(self):
+        return self._organisation.get('subdomain', '')
+
+    # def __print_organisation_telescopes(org_subdomain: str, api: str = 'main'):
+    #     click.echo(f" • Here is a list of existing telescopes for organisation '{org_subdomain}':")
+    #     kwargs = build_endpoint_kwargs(api, org_subdomain)
+    #     telescope_list, error = ArcsecondAPI.telescopes(**kwargs).list()
+    #     for telescope in telescope_list:
+    #         s = f" 🔭 {telescope['name']}"
+    #         if telescope.get('alias', ''):
+    #             s += f" a.k.a. {telescope['alias']}"
+    #         s += f" ({telescope['uuid']})"
+    #         click.echo(s)