diff --git a/src/diffusers/pipelines/pipeline_utils.py b/src/diffusers/pipelines/pipeline_utils.py
index e5f822caa0ef..d946383f47ee 100644
--- a/src/diffusers/pipelines/pipeline_utils.py
+++ b/src/diffusers/pipelines/pipeline_utils.py
@@ -376,11 +376,12 @@ def module_is_sequentially_offloaded(module):
             if not is_accelerate_available() or is_accelerate_version("<", "0.14.0"):
                 return False
 
-            return hasattr(module, "_hf_hook") and (
-                isinstance(module._hf_hook, accelerate.hooks.AlignDevicesHook)
-                or hasattr(module._hf_hook, "hooks")
-                and isinstance(module._hf_hook.hooks[0], accelerate.hooks.AlignDevicesHook)
-            )
+            if hasattr(module, "_no_split_modules") and module._no_split_modules is None:
+                return hasattr(module, "_hf_hook") and (
+                    isinstance(module._hf_hook, accelerate.hooks.AlignDevicesHook)
+                    or hasattr(module._hf_hook, "hooks")
+                    and isinstance(module._hf_hook.hooks[0], accelerate.hooks.AlignDevicesHook)
+                )
 
         def module_is_offloaded(module):
             if not is_accelerate_available() or is_accelerate_version("<", "0.17.0.dev0"):