From 7f2493657860b14409d12e7eaca0da4e3ef1b073 Mon Sep 17 00:00:00 2001 From: user1 Date: Tue, 10 Oct 2023 10:10:22 -0700 Subject: [PATCH 1/7] Add ability to mix usage of T2I-Adapter(s) and ControlNet(s). Previously, UNet2DConditional implemnetation onloy allowed use of one or the other. Adds new forward() arg down_intrablock_additional_residuals specifically for T2I-Adapters. If down_intrablock_addtional_residuals is not used, maintains backward compatibility with prior usage of only T2I-Adapter or ControlNet but not both --- src/diffusers/models/unet_2d_condition.py | 34 +++++++++++++++++------ 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 52c3fc141e59..8ab704e6449c 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -778,6 +778,7 @@ def forward( added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, mid_block_additional_residual: Optional[torch.Tensor] = None, + down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None, encoder_attention_mask: Optional[torch.Tensor] = None, return_dict: bool = True, ) -> Union[UNet2DConditionOutput, Tuple]: @@ -802,6 +803,13 @@ def forward( added_cond_kwargs: (`dict`, *optional*): A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that are passed along to the UNet blocks. + down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*) + additional residuals to be added to UNet long skip connections from down blocks to up blocks + for example from ControlNet side model(s) + mid_block_additional_residual (`torch.Tensor`, *optional*) + additional residual to be added to UNet mid block output, for example from ControlNet side model + down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*) + additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s) Returns: [`~models.unet_2d_condition.UNet2DConditionOutput`] or `tuple`: @@ -977,15 +985,24 @@ def forward( lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None - is_adapter = mid_block_additional_residual is None and down_block_additional_residuals is not None + # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets + is_adapter = down_intrablock_additional_residuals is not None + # maintain backward compatibility for legacy usage, where + # T2I-Adapter and ControlNet both use down_block_additional_residuals arg + # but can only use one or the other + if not is_adapter: + # testing for legacy usage + if mid_block_additional_residual is None and down_block_additional_residuals is not None: + down_intrablock_additional_residuals = down_block_additional_residuals + is_adapter = True down_block_res_samples = (sample,) for downsample_block in self.down_blocks: if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: # For t2i-adapter CrossAttnDownBlock2D additional_residuals = {} - if is_adapter and len(down_block_additional_residuals) > 0: - additional_residuals["additional_residuals"] = down_block_additional_residuals.pop(0) + if is_adapter and len(down_intrablock_additional_residuals) > 0: + additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0) sample, res_samples = downsample_block( hidden_states=sample, @@ -998,9 +1015,8 @@ def forward( ) else: sample, res_samples = downsample_block(hidden_states=sample, temb=emb, scale=lora_scale) - - if is_adapter and len(down_block_additional_residuals) > 0: - sample += down_block_additional_residuals.pop(0) + if is_adapter and len(down_intrablock_additional_residuals) > 0: + sample += down_intrablock_additional_residuals.pop(0) down_block_res_samples += res_samples @@ -1028,10 +1044,10 @@ def forward( # To support T2I-Adapter-XL if ( is_adapter - and len(down_block_additional_residuals) > 0 - and sample.shape == down_block_additional_residuals[0].shape + and len(down_intrablock_additional_residuals) > 0 + and sample.shape == down_intrablock_additional_residuals[0].shape ): - sample += down_block_additional_residuals.pop(0) + sample += down_intrablock_additional_residuals.pop(0) if is_controlnet: sample = sample + mid_block_additional_residual From 40316c9b419e8989750fec4b02f839841f58f896 Mon Sep 17 00:00:00 2001 From: Gregg Helt Date: Tue, 10 Oct 2023 16:07:45 -0700 Subject: [PATCH 2/7] Improving forward() arg docs in src/diffusers/models/unet_2d_condition.py Co-authored-by: psychedelicious <4822129+psychedelicious@users.noreply.github.com> --- src/diffusers/models/unet_2d_condition.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 8ab704e6449c..0822756ea371 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -803,12 +803,12 @@ def forward( added_cond_kwargs: (`dict`, *optional*): A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that are passed along to the UNet blocks. - down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*) + down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*): additional residuals to be added to UNet long skip connections from down blocks to up blocks for example from ControlNet side model(s) - mid_block_additional_residual (`torch.Tensor`, *optional*) + mid_block_additional_residual (`torch.Tensor`, *optional*): additional residual to be added to UNet mid block output, for example from ControlNet side model - down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*) + down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*): additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s) Returns: From ba716ed6a25b60081ea416df6fc297eb9bc19485 Mon Sep 17 00:00:00 2001 From: Gregg Helt Date: Sun, 15 Oct 2023 12:20:48 -0700 Subject: [PATCH 3/7] Add deprecation warning if down_block_additional_residues is used for T2I-Adapter (intrablock residuals) Co-authored-by: Patrick von Platen --- src/diffusers/models/unet_2d_condition.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 0822756ea371..57f896e841e3 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -990,11 +990,10 @@ def forward( # maintain backward compatibility for legacy usage, where # T2I-Adapter and ControlNet both use down_block_additional_residuals arg # but can only use one or the other - if not is_adapter: - # testing for legacy usage - if mid_block_additional_residual is None and down_block_additional_residuals is not None: - down_intrablock_additional_residuals = down_block_additional_residuals - is_adapter = True + if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None: + deprecate("T2I should not use down_block_additional_residuals", "1.3.0", "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated in will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", standard_warn=False) <- let's deprecate using "down_block_additional_residuals" for T2I adapters + down_intrablock_additional_residuals = down_block_additional_residuals + is_adapter = True down_block_res_samples = (sample,) for downsample_block in self.down_blocks: From 2c2da0b66b6e644dd30045f921494e54e31545d2 Mon Sep 17 00:00:00 2001 From: user1 Date: Sun, 15 Oct 2023 13:39:58 -0700 Subject: [PATCH 4/7] Oops my bad, fixing last commit. --- src/diffusers/models/unet_2d_condition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 57f896e841e3..5859e79186b8 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -991,7 +991,7 @@ def forward( # T2I-Adapter and ControlNet both use down_block_additional_residuals arg # but can only use one or the other if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None: - deprecate("T2I should not use down_block_additional_residuals", "1.3.0", "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated in will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", standard_warn=False) <- let's deprecate using "down_block_additional_residuals" for T2I adapters + deprecate("T2I should not use down_block_additional_residuals", "1.3.0", "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated in will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", standard_warn=False) down_intrablock_additional_residuals = down_block_additional_residuals is_adapter = True From 107069a4ade2464ba103f4ba66032ab408b47147 Mon Sep 17 00:00:00 2001 From: user1 Date: Sun, 15 Oct 2023 13:46:14 -0700 Subject: [PATCH 5/7] Added import of diffusers utils.deprecate --- src/diffusers/models/unet_2d_condition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index b75b4a048c88..88d6091198a6 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -20,7 +20,7 @@ from ..configuration_utils import ConfigMixin, register_to_config from ..loaders import UNet2DConditionLoadersMixin -from ..utils import USE_PEFT_BACKEND, BaseOutput, logging, scale_lora_layers, unscale_lora_layers +from ..utils import USE_PEFT_BACKEND, BaseOutput, logging, deprecate, scale_lora_layers, unscale_lora_layers from .activations import get_activation from .attention_processor import ( ADDED_KV_ATTENTION_PROCESSORS, From d9c029873b7f9b9a9bc977745d93474f78124cd0 Mon Sep 17 00:00:00 2001 From: user1 Date: Sun, 15 Oct 2023 19:55:38 -0700 Subject: [PATCH 6/7] Conform to max line length --- src/diffusers/models/unet_2d_condition.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/diffusers/models/unet_2d_condition.py b/src/diffusers/models/unet_2d_condition.py index 88d6091198a6..06421305c301 100644 --- a/src/diffusers/models/unet_2d_condition.py +++ b/src/diffusers/models/unet_2d_condition.py @@ -1014,7 +1014,12 @@ def forward( # T2I-Adapter and ControlNet both use down_block_additional_residuals arg # but can only use one or the other if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None: - deprecate("T2I should not use down_block_additional_residuals", "1.3.0", "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated in will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", standard_warn=False) + deprecate("T2I should not use down_block_additional_residuals", + "1.3.0", + "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \ + and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \ + for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", + standard_warn=False) down_intrablock_additional_residuals = down_block_additional_residuals is_adapter = True From 671cff5153fa2f4ba3d42e48d9d450cda8c6d3be Mon Sep 17 00:00:00 2001 From: user1 Date: Mon, 16 Oct 2023 03:15:34 -0700 Subject: [PATCH 7/7] Modifying T2I-Adapter pipelines to reflect change to UNet forward() arg for T2I-Adapter residuals. --- .../t2i_adapter/pipeline_stable_diffusion_adapter.py | 2 +- .../t2i_adapter/pipeline_stable_diffusion_xl_adapter.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py index 0c7120c5b3ec..dca9e5fc3de2 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py @@ -813,7 +813,7 @@ def __call__( t, encoder_hidden_states=prompt_embeds, cross_attention_kwargs=cross_attention_kwargs, - down_block_additional_residuals=[state.clone() for state in adapter_state], + down_intrablock_additional_residuals=[state.clone() for state in adapter_state], ).sample # perform guidance diff --git a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py index b31d478a9d67..d4272696c23b 100644 --- a/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +++ b/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -975,9 +975,9 @@ def __call__( added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} if i < int(num_inference_steps * adapter_conditioning_factor): - down_block_additional_residuals = [state.clone() for state in adapter_state] + down_intrablock_additional_residuals = [state.clone() for state in adapter_state] else: - down_block_additional_residuals = None + down_intrablock_additional_residuals = None noise_pred = self.unet( latent_model_input, @@ -986,7 +986,7 @@ def __call__( cross_attention_kwargs=cross_attention_kwargs, added_cond_kwargs=added_cond_kwargs, return_dict=False, - down_block_additional_residuals=down_block_additional_residuals, + down_intrablock_additional_residuals=down_intrablock_additional_residuals, )[0] # perform guidance