Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding verbosity option in version downloads #166

Closed
wants to merge 2 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions roboflow/core/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,13 +163,14 @@ def __wait_if_generating(self, recurse=False):
sys.stdout.flush()
return

def download(self, model_format=None, location=None, overwrite: bool = True):
def download(self, model_format=None, location=None, overwrite: bool = True, verbose: bool = True):
"""
Download and extract a ZIP of a version's dataset in a given format

:param model_format: A format to use for downloading
:param location: An optional path for saving the file
:param overwrite: An optional flag to prevent dataset overwrite when dataset is already downloaded
:param verbose: An optional flag to enable or disable verbose output

:return: Dataset
"""
Expand Down Expand Up @@ -217,8 +218,8 @@ def download(self, model_format=None, location=None, overwrite: bool = True):
except requests.exceptions.JSONDecodeError:
response.raise_for_status()

self.__download_zip(link, location, model_format)
self.__extract_zip(location, model_format)
self.__download_zip(link, location, model_format, verbose)
self.__extract_zip(location, model_format, verbose)
self.__reformat_yaml(location, model_format)

return Dataset(self.name, self.version, model_format, os.path.abspath(location))
Expand Down Expand Up @@ -606,13 +607,14 @@ def deploy(self, model_type: str, model_path: str) -> None:
except Exception as e:
print(f"An error occured when uploading the model: {e}")

def __download_zip(self, link, location, format):
def __download_zip(self, link, location, format, verbose):
"""
Download a dataset's zip file from the given URL and save it in the desired location

:param location: link the URL of the remote zip file
:param location: filepath of the data directory to save the zip file to
:param format: the format identifier string
:param verbose: enable or disable verbose output

:return None:
"""
Expand All @@ -631,28 +633,37 @@ def bar_progress(current, total, width=80):
sys.stdout.flush()

try:
wget.download(link, out=location + "/roboflow.zip", bar=bar_progress)
if verbose:
wget.download(link, out=location + "/roboflow.zip", bar=bar_progress)
else:
wget.download(link, out=location + "/roboflow.zip", bar=None)
except Exception as e:
print(f"Error when trying to download dataset @ {link}")
raise e
sys.stdout.write("\n")
sys.stdout.flush()

def __extract_zip(self, location, format):
def __extract_zip(self, location, format, verbose):
"""
This simply extracts the contents of a downloaded zip file and then deletes the zip

:param location: filepath of the data directory that contains the zip file
:param format: the format identifier string
:param verbose: enable or disable verbose output

:return None:
:raises RuntimeError:
"""
with zipfile.ZipFile(location + "/roboflow.zip", "r") as zip_ref:
for member in tqdm(
zip_ref.infolist(),
desc=f"Extracting Dataset Version Zip to {location} in {format}:",
):
if verbose:
zip_progress = tqdm(
zip_ref.infolist(),
desc=f"Extracting Dataset Version Zip to {location} in {format}:",
)
else:
zip_progress = zip_ref.infolist()

for member in zip_progress:
try:
zip_ref.extract(member, location)
except zipfile.error:
Expand Down
Loading