Revert "[Wuerstchen] fix fp16 training and correct lora args (#6245)"

This reverts commit 0bb9cf0.

Author: sayakpaul

examples/wuerstchen/text_to_image/train_text_to_image_lora_prior.py

@@ -527,17 +527,9 @@ def deepspeed_zero_init_disabled_context_manager():
 
     # lora attn processor
     prior_lora_config = LoraConfig(
-        r=args.rank,
-        lora_alpha=args.rank,
-        target_modules=["to_k", "to_q", "to_v", "to_out.0", "add_k_proj", "add_v_proj"],
+        r=args.rank, target_modules=["to_k", "to_q", "to_v", "to_out.0", "add_k_proj", "add_v_proj"]
     )
-    # Add adapter and make sure the trainable params are in float32.
     prior.add_adapter(prior_lora_config)
-    if args.mixed_precision == "fp16":
-        for param in prior.parameters():
-            # only upcast trainable parameters (LoRA) into fp32
-            if param.requires_grad:
-                param.data = param.to(torch.float32)
 
     # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format
     def save_model_hook(models, weights, output_dir):