From 5f0f4689181189cb3d3db44229ce78428f82a787 Mon Sep 17 00:00:00 2001 From: Aryan Date: Thu, 25 Jul 2024 11:59:00 +0200 Subject: [PATCH 1/3] fix step index out of bounds --- src/diffusers/pipelines/free_init_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/free_init_utils.py b/src/diffusers/pipelines/free_init_utils.py index 4f7965a038c5..e0d1bc5ba770 100644 --- a/src/diffusers/pipelines/free_init_utils.py +++ b/src/diffusers/pipelines/free_init_utils.py @@ -180,6 +180,6 @@ def _apply_free_init( num_inference_steps = max( 1, int(num_inference_steps / self._free_init_num_iters * (free_init_iteration + 1)) ) - self.scheduler.set_timesteps(num_inference_steps, device=device) + self.scheduler.set_timesteps(num_inference_steps, device=device) return latents, self.scheduler.timesteps From 427db99d5a292025907b28bab3eaa0506e06913d Mon Sep 17 00:00:00 2001 From: Aryan Date: Thu, 25 Jul 2024 22:42:03 +0200 Subject: [PATCH 2/3] add test for free_init with different schedulers --- .../pipelines/animatediff/test_animatediff.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/pipelines/animatediff/test_animatediff.py b/tests/pipelines/animatediff/test_animatediff.py index 66b7177f4be1..dd636b7ce669 100644 --- a/tests/pipelines/animatediff/test_animatediff.py +++ b/tests/pipelines/animatediff/test_animatediff.py @@ -10,6 +10,8 @@ AnimateDiffPipeline, AutoencoderKL, DDIMScheduler, + DPMSolverMultistepScheduler, + LCMScheduler, MotionAdapter, StableDiffusionPipeline, UNet2DConditionModel, @@ -353,6 +355,52 @@ def test_free_init(self): "Disabling of FreeInit should lead to results similar to the default pipeline results", ) + def test_free_init_with_schedulers(self): + components = self.get_dummy_components() + pipe: AnimateDiffPipeline = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs_normal = self.get_dummy_inputs(torch_device) + frames_normal = pipe(**inputs_normal).frames[0] + + schedulers_to_test = [ + DPMSolverMultistepScheduler.from_config( + components["scheduler"].config, + timestep_spacing="linspace", + beta_schedule="linear", + algorithm_type="dpmsolver++", + steps_offset=1, + clip_sample=False, + ), + LCMScheduler.from_config( + components["scheduler"].config, + timestep_spacing="linspace", + beta_schedule="linear", + steps_offset=1, + clip_sample=False, + ), + ] + components.pop("scheduler") + + for scheduler in schedulers_to_test: + components["scheduler"] = scheduler + pipe: AnimateDiffPipeline = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + pipe.enable_free_init(num_iters=2, use_fast_sampling=False) + + inputs = self.get_dummy_inputs(torch_device) + frames_enable_free_init = pipe(**inputs).frames[0] + sum_enabled = np.abs(to_np(frames_normal) - to_np(frames_enable_free_init)).sum() + + self.assertGreater( + sum_enabled, + 1e1, + "Enabling of FreeInit should lead to results different from the default pipeline results", + ) + @unittest.skipIf( torch_device != "cuda" or not is_xformers_available(), reason="XFormers attention is only available with CUDA and `xformers` installed", From e3e268fc83e89ce10e510457c7140a456d49ca61 Mon Sep 17 00:00:00 2001 From: Aryan Date: Fri, 26 Jul 2024 12:38:52 +0200 Subject: [PATCH 3/3] add test to vid2vid and pia --- src/diffusers/pipelines/free_init_utils.py | 4 +- .../test_animatediff_video2video.py | 48 +++++++++++++++++++ tests/pipelines/pia/test_pia.py | 48 +++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) diff --git a/src/diffusers/pipelines/free_init_utils.py b/src/diffusers/pipelines/free_init_utils.py index e0d1bc5ba770..1fb67592ca4f 100644 --- a/src/diffusers/pipelines/free_init_utils.py +++ b/src/diffusers/pipelines/free_init_utils.py @@ -181,5 +181,7 @@ def _apply_free_init( 1, int(num_inference_steps / self._free_init_num_iters * (free_init_iteration + 1)) ) - self.scheduler.set_timesteps(num_inference_steps, device=device) + if num_inference_steps > 0: + self.scheduler.set_timesteps(num_inference_steps, device=device) + return latents, self.scheduler.timesteps diff --git a/tests/pipelines/animatediff/test_animatediff_video2video.py b/tests/pipelines/animatediff/test_animatediff_video2video.py index 56bbd69a7c9b..ced042b4a702 100644 --- a/tests/pipelines/animatediff/test_animatediff_video2video.py +++ b/tests/pipelines/animatediff/test_animatediff_video2video.py @@ -10,6 +10,8 @@ AnimateDiffVideoToVideoPipeline, AutoencoderKL, DDIMScheduler, + DPMSolverMultistepScheduler, + LCMScheduler, MotionAdapter, StableDiffusionPipeline, UNet2DConditionModel, @@ -380,3 +382,49 @@ def test_free_init(self): 1e-4, "Disabling of FreeInit should lead to results similar to the default pipeline results", ) + + def test_free_init_with_schedulers(self): + components = self.get_dummy_components() + pipe: AnimateDiffVideoToVideoPipeline = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs_normal = self.get_dummy_inputs(torch_device) + frames_normal = pipe(**inputs_normal).frames[0] + + schedulers_to_test = [ + DPMSolverMultistepScheduler.from_config( + components["scheduler"].config, + timestep_spacing="linspace", + beta_schedule="linear", + algorithm_type="dpmsolver++", + steps_offset=1, + clip_sample=False, + ), + LCMScheduler.from_config( + components["scheduler"].config, + timestep_spacing="linspace", + beta_schedule="linear", + steps_offset=1, + clip_sample=False, + ), + ] + components.pop("scheduler") + + for scheduler in schedulers_to_test: + components["scheduler"] = scheduler + pipe: AnimateDiffVideoToVideoPipeline = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + pipe.enable_free_init(num_iters=2, use_fast_sampling=False) + + inputs = self.get_dummy_inputs(torch_device) + frames_enable_free_init = pipe(**inputs).frames[0] + sum_enabled = np.abs(to_np(frames_normal) - to_np(frames_enable_free_init)).sum() + + self.assertGreater( + sum_enabled, + 1e1, + "Enabling of FreeInit should lead to results different from the default pipeline results", + ) diff --git a/tests/pipelines/pia/test_pia.py b/tests/pipelines/pia/test_pia.py index 4cf11a8e9c4e..83f550f30b23 100644 --- a/tests/pipelines/pia/test_pia.py +++ b/tests/pipelines/pia/test_pia.py @@ -9,6 +9,8 @@ from diffusers import ( AutoencoderKL, DDIMScheduler, + DPMSolverMultistepScheduler, + LCMScheduler, MotionAdapter, PIAPipeline, StableDiffusionPipeline, @@ -360,6 +362,52 @@ def test_free_init(self): "Disabling of FreeInit should lead to results similar to the default pipeline results", ) + def test_free_init_with_schedulers(self): + components = self.get_dummy_components() + pipe: PIAPipeline = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs_normal = self.get_dummy_inputs(torch_device) + frames_normal = pipe(**inputs_normal).frames[0] + + schedulers_to_test = [ + DPMSolverMultistepScheduler.from_config( + components["scheduler"].config, + timestep_spacing="linspace", + beta_schedule="linear", + algorithm_type="dpmsolver++", + steps_offset=1, + clip_sample=False, + ), + LCMScheduler.from_config( + components["scheduler"].config, + timestep_spacing="linspace", + beta_schedule="linear", + steps_offset=1, + clip_sample=False, + ), + ] + components.pop("scheduler") + + for scheduler in schedulers_to_test: + components["scheduler"] = scheduler + pipe: PIAPipeline = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + pipe.enable_free_init(num_iters=2, use_fast_sampling=False) + + inputs = self.get_dummy_inputs(torch_device) + frames_enable_free_init = pipe(**inputs).frames[0] + sum_enabled = np.abs(to_np(frames_normal) - to_np(frames_enable_free_init)).sum() + + self.assertGreater( + sum_enabled, + 1e1, + "Enabling of FreeInit should lead to results different from the default pipeline results", + ) + @unittest.skipIf( torch_device != "cuda" or not is_xformers_available(), reason="XFormers attention is only available with CUDA and `xformers` installed",