fix for dreambooth lora sdxl

Author: sayakpaul

examples/dreambooth/train_dreambooth_lora_sdxl.py

@@ -1024,6 +1024,12 @@ def main(args):
         text_encoder_one.add_adapter(text_lora_config)
         text_encoder_two.add_adapter(text_lora_config)
 
+    # Make sure the trainable params are in float32.
+    for model in [unet, text_encoder_one, text_encoder_two]:
+        for param in model.parameters():
+            if param.requires_grad:
+                param.data = param.to(torch.float32)
+
     # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format
     def save_model_hook(models, weights, output_dir):
         if accelerator.is_main_process: