Fix LTX frame rate for rope interpolation scale calculation (#244)

a-r-r-o-w · Jan 24, 2025 · e0ced4f · e0ced4f
1 parent 3eb9f8a
commit e0ced4f
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/finetrainers/models/ltx_video/lora.py b/finetrainers/models/ltx_video/lora.py
@@ -205,7 +205,10 @@ def forward_pass(
     **kwargs,
 ) -> torch.Tensor:
     # TODO(aryan): make configurable
-    rope_interpolation_scale = [1 / 25, 32, 32]
+    frame_rate = 25
+    latent_frame_rate = frame_rate / 8
+    spatial_compression_ratio = 32
+    rope_interpolation_scale = [1 / latent_frame_rate, spatial_compression_ratio, spatial_compression_ratio]
 
     denoised_latents = transformer(
         hidden_states=noisy_latents,

diff --git a/finetrainers/trainer.py b/finetrainers/trainer.py
@@ -549,10 +549,10 @@ def train(self) -> None:
         if self.vae_config is None:
             # If we've precomputed conditions and latents already, and are now re-using it, we will never load
             # the VAE so self.vae_config will not be set. So, we need to load it here.
-            vae_cls_name = resolve_vae_cls_from_ckpt_path(
+            vae_cls = resolve_vae_cls_from_ckpt_path(
                 self.args.pretrained_model_name_or_path, revision=self.args.revision, cache_dir=self.args.cache_dir
             )
-            vae_config = vae_cls_name.load_config(
+            vae_config = vae_cls.load_config(
                 self.args.pretrained_model_name_or_path,
                 subfolder="vae",
                 revision=self.args.revision,