From f1a131f2211d04eaa257f32607a45e79af261baf Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 29 Jan 2025 14:14:02 +0530 Subject: [PATCH 1/4] fix: model card info. --- finetrainers/trainer.py | 7 +++++++ finetrainers/utils/hub_utils.py | 15 +++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py index 153e4069..56c4866d 100644 --- a/finetrainers/trainer.py +++ b/finetrainers/trainer.py @@ -863,6 +863,13 @@ def validate(self, step: int, final_validation: bool = False) -> None: if num_validation_samples == 0: logger.warning("No validation samples found. Skipping validation.") + if accelerator.is_main_process: + save_model_card( + args=self.args, + repo_id=self.state.repo_id, + videos=None, + validation_prompts=None, + ) return self.transformer.eval() diff --git a/finetrainers/utils/hub_utils.py b/finetrainers/utils/hub_utils.py index ea1a16eb..ef865407 100644 --- a/finetrainers/utils/hub_utils.py +++ b/finetrainers/utils/hub_utils.py @@ -28,17 +28,20 @@ def save_model_card( } ) + training_type = "Full" if args.training_type == "full-finetune" else "LoRA" model_description = f""" -# LoRA Finetune +# {training_type} Finetune ## Model description -This is a lora finetune of model: `{args.pretrained_model_name_or_path}`. +This is a {training_type.lower()} finetune of model: `{args.pretrained_model_name_or_path}`. The model was trained using [`finetrainers`](https://github.com/a-r-r-o-w/finetrainers). +`id_token` used: {args.id_token} (if it's not `None`, it should be used in the prompts.) + ## Download model [Download LoRA]({repo_id}/tree/main) in the Files & Versions tab. @@ -53,7 +56,7 @@ def save_model_card( For more details, including weighting, merging and fusing LoRAs, check the [documentation](https://huggingface.co/docs/diffusers/main/en/using-diffusers/loading_adapters) on loading LoRAs in diffusers. """ - if wandb.run.url: + if wandb.run and wandb.run.url: model_description += f""" Find out the wandb run URL and training configurations [here]({wandb.run.url}). """ @@ -69,9 +72,13 @@ def save_model_card( "text-to-video", "diffusers-training", "diffusers", - "lora", + "finetrainers", "template:sd-lora", ] + if training_type == "Full": + tags.append("full-finetune") + else: + tags.append("lora") model_card = populate_model_card(model_card, tags=tags) model_card.save(os.path.join(args.output_dir, "README.md")) From 3feca30b11c969611a8e57cf5cd73df21c40a9f9 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 29 Jan 2025 15:13:05 +0530 Subject: [PATCH 2/4] always ensure loading takes place on the main process first. --- finetrainers/trainer.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py index 56c4866d..2a7f2261 100644 --- a/finetrainers/trainer.py +++ b/finetrainers/trainer.py @@ -138,9 +138,12 @@ def prepare_models(self) -> None: load_components_kwargs = self._get_load_components_kwargs() condition_components, latent_components, diffusion_components = {}, {}, {} if not self.args.precompute_conditions: - condition_components = self.model_config["load_condition_models"](**load_components_kwargs) - latent_components = self.model_config["load_latent_models"](**load_components_kwargs) - diffusion_components = self.model_config["load_diffusion_models"](**load_components_kwargs) + # To download the model files first on the main process (if not already present) + # and then load the cached files afterward from the other processes. + with self.state.accelerator.main_process_first(): + condition_components = self.model_config["load_condition_models"](**load_components_kwargs) + latent_components = self.model_config["load_latent_models"](**load_components_kwargs) + diffusion_components = self.model_config["load_diffusion_models"](**load_components_kwargs) components = {} components.update(condition_components) @@ -204,7 +207,8 @@ def collate_fn(batch): logger.info("Precomputed conditions and latents not found. Running precomputation.") # At this point, no models are loaded, so we need to load and precompute conditions and latents - condition_components = self.model_config["load_condition_models"](**self._get_load_components_kwargs()) + with self.state.accelerator.main_process_first(): + condition_components = self.model_config["load_condition_models"](**self._get_load_components_kwargs()) self._set_components(condition_components) self._move_components_to_device() self._disable_grad_for_components([self.text_encoder, self.text_encoder_2, self.text_encoder_3]) @@ -258,7 +262,8 @@ def collate_fn(batch): torch.cuda.reset_peak_memory_stats(accelerator.device) # Precompute latents - latent_components = self.model_config["load_latent_models"](**self._get_load_components_kwargs()) + with self.state.accelerator.main_process_first(): + latent_components = self.model_config["load_latent_models"](**self._get_load_components_kwargs()) self._set_components(latent_components) self._move_components_to_device() self._disable_grad_for_components([self.vae]) @@ -319,7 +324,8 @@ def collate_fn(batch): def prepare_trainable_parameters(self) -> None: logger.info("Initializing trainable parameters") - diffusion_components = self.model_config["load_diffusion_models"](**self._get_load_components_kwargs()) + with self.state.accelerator.main_process_first(): + diffusion_components = self.model_config["load_diffusion_models"](**self._get_load_components_kwargs()) self._set_components(diffusion_components) components = [self.text_encoder, self.text_encoder_2, self.text_encoder_3, self.vae] From 3de90c44618b09a3411ce36ff0dae56fdc7d360c Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 29 Jan 2025 15:17:27 +0530 Subject: [PATCH 3/4] revert changes --- finetrainers/trainer.py | 7 ------- finetrainers/utils/hub_utils.py | 16 +++++----------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py index 2a7f2261..8eb674ec 100644 --- a/finetrainers/trainer.py +++ b/finetrainers/trainer.py @@ -869,13 +869,6 @@ def validate(self, step: int, final_validation: bool = False) -> None: if num_validation_samples == 0: logger.warning("No validation samples found. Skipping validation.") - if accelerator.is_main_process: - save_model_card( - args=self.args, - repo_id=self.state.repo_id, - videos=None, - validation_prompts=None, - ) return self.transformer.eval() diff --git a/finetrainers/utils/hub_utils.py b/finetrainers/utils/hub_utils.py index ef865407..2d6728e9 100644 --- a/finetrainers/utils/hub_utils.py +++ b/finetrainers/utils/hub_utils.py @@ -1,3 +1,4 @@ + import os from typing import List, Union @@ -28,20 +29,17 @@ def save_model_card( } ) - training_type = "Full" if args.training_type == "full-finetune" else "LoRA" model_description = f""" -# {training_type} Finetune +# LoRA Finetune ## Model description -This is a {training_type.lower()} finetune of model: `{args.pretrained_model_name_or_path}`. +This is a lora finetune of model: `{args.pretrained_model_name_or_path}`. The model was trained using [`finetrainers`](https://github.com/a-r-r-o-w/finetrainers). -`id_token` used: {args.id_token} (if it's not `None`, it should be used in the prompts.) - ## Download model [Download LoRA]({repo_id}/tree/main) in the Files & Versions tab. @@ -56,7 +54,7 @@ def save_model_card( For more details, including weighting, merging and fusing LoRAs, check the [documentation](https://huggingface.co/docs/diffusers/main/en/using-diffusers/loading_adapters) on loading LoRAs in diffusers. """ - if wandb.run and wandb.run.url: + if wandb.run.url: model_description += f""" Find out the wandb run URL and training configurations [here]({wandb.run.url}). """ @@ -72,13 +70,9 @@ def save_model_card( "text-to-video", "diffusers-training", "diffusers", - "finetrainers", + "lora", "template:sd-lora", ] - if training_type == "Full": - tags.append("full-finetune") - else: - tags.append("lora") model_card = populate_model_card(model_card, tags=tags) model_card.save(os.path.join(args.output_dir, "README.md")) From 55f03824a8dc00ac71fcc2bd14eaea36119e262e Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 29 Jan 2025 15:18:07 +0530 Subject: [PATCH 4/4] revert --- finetrainers/utils/hub_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/finetrainers/utils/hub_utils.py b/finetrainers/utils/hub_utils.py index 2d6728e9..ea1a16eb 100644 --- a/finetrainers/utils/hub_utils.py +++ b/finetrainers/utils/hub_utils.py @@ -1,4 +1,3 @@ - import os from typing import List, Union