From d08662d6433b962240e65098ffb2ea3e5f98210e Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Thu, 10 Nov 2022 19:17:33 +0100
Subject: [PATCH 01/13] up

---
 src/diffusers/pipeline_utils.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index a194f3eb34d9..5f411aa4974f 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -709,5 +709,14 @@ def progress_bar(self, iterable):
 
         return tqdm(iterable, **self._progress_bar_config)
 
+    def set_scheduler(self, scheduler_type=Union[str, Dict[str, str]]):
+        schedulers = [k for k, c in self.components if isinstance(c, SchedulerMixin)]
+        if len(schedulers) > 1 and isinstance(scheduler_type, str):
+           raise ValueError("Ambigous")
+    
+        scheduler_type = scheduler_type if isinstance(scheduler_type, dict) else {schedulers[0]: scheduler_type}
+
+        # .. load and set all schedulers
+
     def set_progress_bar_config(self, **kwargs):
         self._progress_bar_config = kwargs

From bc59fe84a1e52ac8fb37a104b4720267908cc332 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 16:33:23 +0100
Subject: [PATCH 02/13] finish

---
 .../cd4a85374821636f69971f3e1840fc21546339a4  | 101 ++++++++++++++++++
 .../refs/main                                 |   1 +
 .../pipeline.py                               |   1 +
 src/diffusers/pipeline_utils.py               |  63 +++++++++--
 src/diffusers/schedulers/__init__.py          |  22 +++-
 tests/test_pipelines.py                       |  66 ++++++++++++
 6 files changed, 246 insertions(+), 8 deletions(-)
 create mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
 create mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
 create mode 120000 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py

diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4 b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
new file mode 100644
index 000000000000..cd4a85374821
--- /dev/null
+++ b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
@@ -0,0 +1,101 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+# limitations under the License.
+
+
+from typing import Optional, Tuple, Union
+
+import torch
+
+from diffusers.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
+
+
+class CustomPipeline(DiffusionPipeline):
+    r"""
+    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
+    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
+
+    Parameters:
+        unet ([`UNet2DModel`]): U-Net architecture to denoise the encoded image.
+        scheduler ([`SchedulerMixin`]):
+            A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of
+            [`DDPMScheduler`], or [`DDIMScheduler`].
+    """
+
+    def __init__(self, unet, scheduler):
+        super().__init__()
+        self.register_modules(unet=unet, scheduler=scheduler)
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        batch_size: int = 1,
+        generator: Optional[torch.Generator] = None,
+        num_inference_steps: int = 50,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        **kwargs,
+    ) -> Union[ImagePipelineOutput, Tuple]:
+        r"""
+        Args:
+            batch_size (`int`, *optional*, defaults to 1):
+                The number of images to generate.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            eta (`float`, *optional*, defaults to 0.0):
+                The eta parameter which controls the scale of the variance (0 is DDIM and 1 is one type of DDPM).
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipeline_utils.ImagePipelineOutput`] instead of a plain tuple.
+
+        Returns:
+            [`~pipeline_utils.ImagePipelineOutput`] or `tuple`: [`~pipelines.utils.ImagePipelineOutput`] if
+            `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is a list with the
+            generated images.
+        """
+
+        # Sample gaussian noise to begin loop
+        image = torch.randn(
+            (batch_size, self.unet.in_channels, self.unet.sample_size, self.unet.sample_size),
+            generator=generator,
+        )
+        image = image.to(self.device)
+
+        # set step values
+        self.scheduler.set_timesteps(num_inference_steps)
+
+        for t in self.progress_bar(self.scheduler.timesteps):
+            # 1. predict noise model_output
+            model_output = self.unet(image, t).sample
+
+            # 2. predict previous mean of image x_t-1 and add variance depending on eta
+            # eta corresponds to η in paper and should be between [0, 1]
+            # do x_t -> x_t-1
+            image = self.scheduler.step(model_output, t, image).prev_sample
+
+        image = (image / 2 + 0.5).clamp(0, 1)
+        image = image.cpu().permute(0, 2, 3, 1).numpy()
+        if output_type == "pil":
+            image = self.numpy_to_pil(image)
+
+        if not return_dict:
+            return (image,)
+
+        return ImagePipelineOutput(images=image), "This is a test"
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
new file mode 100644
index 000000000000..b8b710da3d11
--- /dev/null
+++ b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
@@ -0,0 +1 @@
+59d7c3f29a8379f8d9d53e3e111212a3da855e70
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
new file mode 120000
index 000000000000..3bf3ab2bdf65
--- /dev/null
+++ b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
@@ -0,0 +1 @@
+../../blobs/cd4a85374821636f69971f3e1840fc21546339a4
\ No newline at end of file
diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index 5f411aa4974f..b1bda4381fcc 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -34,7 +34,8 @@
 from .dynamic_modules_utils import get_class_from_dynamic_module
 from .hub_utils import http_user_agent
 from .modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT
-from .schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME
+from .schedulers import CLASS_TO_SCHEDULER_TYPE_MAPPING, SCHEDULER_TYPE_TO_CLASS_MAPPING
+from .schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME, SchedulerMixin
 from .utils import (
     CONFIG_NAME,
     DIFFUSERS_CACHE,
@@ -710,13 +711,61 @@ def progress_bar(self, iterable):
         return tqdm(iterable, **self._progress_bar_config)
 
     def set_scheduler(self, scheduler_type=Union[str, Dict[str, str]]):
-        schedulers = [k for k, c in self.components if isinstance(c, SchedulerMixin)]
-        if len(schedulers) > 1 and isinstance(scheduler_type, str):
-           raise ValueError("Ambigous")
-    
-        scheduler_type = scheduler_type if isinstance(scheduler_type, dict) else {schedulers[0]: scheduler_type}
+        schedulers = {k: type(v) for k, v in self.components.items() if isinstance(v, SchedulerMixin)}
+        if isinstance(scheduler_type, str) and len(set(schedulers.values())) > 1:
+            raise ValueError(
+                f"The pipeline {self} contains the schedulers {schedulers}. Please make sure to provide a dictionary"
+                f" that maps the componet names {schedulers.keys()} to scheduler types instead of just one scheduler"
+                f" type {scheduler_type}"
+            )
+        elif isinstance(scheduler_type, dict):
+            is_type_scheduler = {k: k in schedulers for k in scheduler_type.keys()}
+            if not all(is_type_scheduler.values()):
+                raise ValueError(
+                    "The following component names are not schedulers"
+                    f" {[k for k, v in is_type_scheduler.items() if v == False]}. Please make sure to only set new"
+                    f" scheduler types for {schedulers.keys()}."
+                )
+
+        scheduler_mapping = (
+            scheduler_type if isinstance(scheduler_type, dict) else {next(iter(schedulers.keys())): scheduler_type}
+        )
+
+        for component_name, scheduler_type in scheduler_mapping.items():
+            scheduler_class = SCHEDULER_TYPE_TO_CLASS_MAPPING.get(scheduler_type, None)
+            current_scheduler = getattr(self, component_name)
+
+            if scheduler_class is None:
+                raise ValueError(
+                    f"{scheduler_type} does not exist, make sure to chose a scheduler type from"
+                    f" {', '.join(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys())}."
+                )
+
+            if scheduler_class not in current_scheduler._compatible_classes and scheduler_class != type(
+                current_scheduler
+            ):
+                diffusers_library = importlib.import_module(__name__.split(".")[0])
+                _compatible_class_types = [
+                    CLASS_TO_SCHEDULER_TYPE_MAPPING[getattr(diffusers_library, c)]
+                    for c in current_scheduler._compatible_classes
+                ]
+                logger.warn(
+                    f"Changing scheduler from type {CLASS_TO_SCHEDULER_TYPE_MAPPING[type(current_scheduler)]} to an"
+                    f" uncompatible scheduler type {scheduler_type}. This is very likely going to lead to incorrect"
+                    f" predictions when running the pipeline. Make sure to set {component_name} to a scheduler of type"
+                    f" {[' ,'.join(_compatible_class_types)]}."
+                )
 
-        # .. load and set all schedulers
+            scheduler_config = current_scheduler.config
+            scheduler_init_dict, _ = scheduler_class.extract_init_dict(scheduler_config)
+
+            scheduler = scheduler_class(**scheduler_init_dict)
+
+            logger.info(
+                f"Changing scheduler from type {CLASS_TO_SCHEDULER_TYPE_MAPPING[type(current_scheduler)]} to"
+                f" {scheduler_type}."
+            )
+            setattr(self, component_name, scheduler)
 
     def set_progress_bar_config(self, **kwargs):
         self._progress_bar_config = kwargs
diff --git a/src/diffusers/schedulers/__init__.py b/src/diffusers/schedulers/__init__.py
index 6217bfcd6985..5e43ce20a2c5 100644
--- a/src/diffusers/schedulers/__init__.py
+++ b/src/diffusers/schedulers/__init__.py
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+from collections import OrderedDict
 
 from ..utils import is_flax_available, is_scipy_available, is_torch_available
 
@@ -50,3 +50,23 @@
     from .scheduling_lms_discrete import LMSDiscreteScheduler
 else:
     from ..utils.dummy_torch_and_scipy_objects import *  # noqa F403
+
+
+SCHEDULER_TYPE_TO_CLASS_MAPPING = OrderedDict(
+    [
+        ("ddim", DDIMScheduler),
+        ("ddpm", DDPMScheduler),
+        ("dpm-multistep", DPMSolverMultistepScheduler),
+        ("euler-ancestral-discrete", EulerAncestralDiscreteScheduler),
+        ("euler-discrete", EulerDiscreteScheduler),
+        ("ipndm", IPNDMScheduler),
+        ("karras-ve", KarrasVeScheduler),
+        ("pndm", PNDMScheduler),
+        ("repaint", RePaintScheduler),
+        ("score-sde-ve", ScoreSdeVeScheduler),
+        ("score-sde-vp", ScoreSdeVpScheduler),
+        ("vq-diffusion", VQDiffusionScheduler),
+        ("lms-discrete", LMSDiscreteScheduler),
+    ]
+)
+CLASS_TO_SCHEDULER_TYPE_MAPPING = OrderedDict({v: k for k, v in SCHEDULER_TYPE_TO_CLASS_MAPPING.items()})
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 4559d713ed81..55f1609901cd 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -22,6 +22,7 @@
 import numpy as np
 import torch
 
+import diffusers
 import PIL
 from diffusers import (
     AutoencoderKL,
@@ -29,6 +30,10 @@
     DDIMScheduler,
     DDPMPipeline,
     DDPMScheduler,
+    DPMSolverMultistepScheduler,
+    EulerAncestralDiscreteScheduler,
+    EulerDiscreteScheduler,
+    LMSDiscreteScheduler,
     PNDMScheduler,
     StableDiffusionImg2ImgPipeline,
     StableDiffusionInpaintPipelineLegacy,
@@ -398,6 +403,67 @@ def test_components(self):
         assert image_img2img.shape == (1, 32, 32, 3)
         assert image_text2img.shape == (1, 128, 128, 3)
 
+    def test_set_scheduler(self):
+        unet = self.dummy_cond_unet
+        scheduler = PNDMScheduler(skip_prk_steps=True)
+        vae = self.dummy_vae
+        bert = self.dummy_text_encoder
+        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")
+
+        sd = StableDiffusionPipeline(
+            unet=unet,
+            scheduler=scheduler,
+            vae=vae,
+            text_encoder=bert,
+            tokenizer=tokenizer,
+            safety_checker=None,
+            feature_extractor=self.dummy_extractor,
+        )
+
+        sd.set_scheduler("ddim")
+        assert isinstance(sd.scheduler, DDIMScheduler)
+        sd.set_scheduler("ddpm")
+        assert isinstance(sd.scheduler, DDPMScheduler)
+        sd.set_scheduler("pndm")
+        assert isinstance(sd.scheduler, PNDMScheduler)
+        sd.set_scheduler("lms-discrete")
+        assert isinstance(sd.scheduler, LMSDiscreteScheduler)
+        sd.set_scheduler("euler-discrete")
+        assert isinstance(sd.scheduler, EulerDiscreteScheduler)
+        sd.set_scheduler("euler-ancestral-discrete")
+        assert isinstance(sd.scheduler, EulerAncestralDiscreteScheduler)
+        sd.set_scheduler("dpm-multistep")
+        assert isinstance(sd.scheduler, DPMSolverMultistepScheduler)
+
+        sd.set_scheduler({"scheduler": "dpm-multistep"})
+        assert isinstance(sd.scheduler, DPMSolverMultistepScheduler)
+
+        logger = logging.get_logger("diffusers.pipeline_utils")
+        with self.assertRaises(ValueError) as error_1:
+            sd.set_scheduler({"schedule": "dpm-multistep"})
+
+        with self.assertRaises(ValueError) as error_2:
+            sd.set_scheduler({"scheduler": "dpm-multiste"})
+
+        logger.setLevel(diffusers.logging.INFO)
+        with CaptureLogger(logger) as cap_logger:
+            sd.set_scheduler({"scheduler": "dpm-multistep"})
+
+        with CaptureLogger(logger) as cap_logger_warn:
+            sd.set_scheduler({"scheduler": "ipndm"})
+
+        assert (
+            str(error_1.exception)
+            == "The following component names are not schedulers ['schedule']. Please make sure to only set new"
+            " scheduler types for dict_keys(['scheduler'])."
+        )
+        assert "dpm-multiste does not exist, make sure to chose a scheduler type from" in str(error_2.exception)
+        assert cap_logger.out == "Changing scheduler from type dpm-multistep to dpm-multistep.\n"
+        assert (
+            "Changing scheduler from type dpm-multistep to an uncompatible scheduler type ipndm."
+            in cap_logger_warn.out
+        )
+
 
 @slow
 class PipelineSlowTests(unittest.TestCase):

From 4d3c5435bda472219b12af297b2156b58ab8cb98 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 16:33:33 +0100
Subject: [PATCH 03/13] up

---
 .../cd4a85374821636f69971f3e1840fc21546339a4  | 101 ------------------
 .../refs/main                                 |   1 -
 .../pipeline.py                               |   1 -
 3 files changed, 103 deletions(-)
 delete mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
 delete mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
 delete mode 120000 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py

diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4 b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
deleted file mode 100644
index cd4a85374821..000000000000
--- a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
+++ /dev/null
@@ -1,101 +0,0 @@
-# Copyright 2022 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-
-# limitations under the License.
-
-
-from typing import Optional, Tuple, Union
-
-import torch
-
-from diffusers.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
-
-
-class CustomPipeline(DiffusionPipeline):
-    r"""
-    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
-    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
-
-    Parameters:
-        unet ([`UNet2DModel`]): U-Net architecture to denoise the encoded image.
-        scheduler ([`SchedulerMixin`]):
-            A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of
-            [`DDPMScheduler`], or [`DDIMScheduler`].
-    """
-
-    def __init__(self, unet, scheduler):
-        super().__init__()
-        self.register_modules(unet=unet, scheduler=scheduler)
-
-    @torch.no_grad()
-    def __call__(
-        self,
-        batch_size: int = 1,
-        generator: Optional[torch.Generator] = None,
-        num_inference_steps: int = 50,
-        output_type: Optional[str] = "pil",
-        return_dict: bool = True,
-        **kwargs,
-    ) -> Union[ImagePipelineOutput, Tuple]:
-        r"""
-        Args:
-            batch_size (`int`, *optional*, defaults to 1):
-                The number of images to generate.
-            generator (`torch.Generator`, *optional*):
-                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
-                deterministic.
-            eta (`float`, *optional*, defaults to 0.0):
-                The eta parameter which controls the scale of the variance (0 is DDIM and 1 is one type of DDPM).
-            num_inference_steps (`int`, *optional*, defaults to 50):
-                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
-                expense of slower inference.
-            output_type (`str`, *optional*, defaults to `"pil"`):
-                The output format of the generate image. Choose between
-                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
-            return_dict (`bool`, *optional*, defaults to `True`):
-                Whether or not to return a [`~pipeline_utils.ImagePipelineOutput`] instead of a plain tuple.
-
-        Returns:
-            [`~pipeline_utils.ImagePipelineOutput`] or `tuple`: [`~pipelines.utils.ImagePipelineOutput`] if
-            `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is a list with the
-            generated images.
-        """
-
-        # Sample gaussian noise to begin loop
-        image = torch.randn(
-            (batch_size, self.unet.in_channels, self.unet.sample_size, self.unet.sample_size),
-            generator=generator,
-        )
-        image = image.to(self.device)
-
-        # set step values
-        self.scheduler.set_timesteps(num_inference_steps)
-
-        for t in self.progress_bar(self.scheduler.timesteps):
-            # 1. predict noise model_output
-            model_output = self.unet(image, t).sample
-
-            # 2. predict previous mean of image x_t-1 and add variance depending on eta
-            # eta corresponds to η in paper and should be between [0, 1]
-            # do x_t -> x_t-1
-            image = self.scheduler.step(model_output, t, image).prev_sample
-
-        image = (image / 2 + 0.5).clamp(0, 1)
-        image = image.cpu().permute(0, 2, 3, 1).numpy()
-        if output_type == "pil":
-            image = self.numpy_to_pil(image)
-
-        if not return_dict:
-            return (image,)
-
-        return ImagePipelineOutput(images=image), "This is a test"
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
deleted file mode 100644
index b8b710da3d11..000000000000
--- a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
+++ /dev/null
@@ -1 +0,0 @@
-59d7c3f29a8379f8d9d53e3e111212a3da855e70
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
deleted file mode 120000
index 3bf3ab2bdf65..000000000000
--- a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
+++ /dev/null
@@ -1 +0,0 @@
-../../blobs/cd4a85374821636f69971f3e1840fc21546339a4
\ No newline at end of file

From 86ad281aaf5319359f9356109e2013c7b4d19b37 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 16:38:16 +0100
Subject: [PATCH 04/13] uP

---
 .../pipelines/stable_diffusion/test_stable_diffusion.py  | 8 +++-----
 .../stable_diffusion/test_stable_diffusion_img2img.py    | 7 +++----
 .../stable_diffusion/test_stable_diffusion_inpaint.py    | 7 +++----
 .../test_stable_diffusion_inpaint_legacy.py              | 9 ++-------
 4 files changed, 11 insertions(+), 20 deletions(-)

diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py
index 6e1071124cb7..2e085cc28d03 100644
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py
@@ -651,9 +651,8 @@ def test_stable_diffusion(self):
         assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
 
     def test_stable_diffusion_fast_ddim(self):
-        scheduler = DDIMScheduler.from_config("CompVis/stable-diffusion-v1-1", subfolder="scheduler")
-
-        sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1", scheduler=scheduler)
+        sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1")
+        sd_pipe.set_scheduler("ddim")
         sd_pipe = sd_pipe.to(torch_device)
         sd_pipe.set_progress_bar_config(disable=None)
 
@@ -674,8 +673,7 @@ def test_lms_stable_diffusion_pipeline(self):
         model_id = "CompVis/stable-diffusion-v1-1"
         pipe = StableDiffusionPipeline.from_pretrained(model_id).to(torch_device)
         pipe.set_progress_bar_config(disable=None)
-        scheduler = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler")
-        pipe.scheduler = scheduler
+        pipe.set_scheduler("lms-discrete")
 
         prompt = "a photograph of an astronaut riding a horse"
         generator = torch.Generator(device=torch_device).manual_seed(0)
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py
index 6d5c6feab5bc..a4c9b3b74ae3 100644
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py
@@ -519,12 +519,11 @@ def test_stable_diffusion_img2img_pipeline_k_lms(self):
         )
 
         model_id = "CompVis/stable-diffusion-v1-4"
-        lms = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler")
         pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
             model_id,
-            scheduler=lms,
             safety_checker=None,
         )
+        pipe.set_scheduler("lms-discrete")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing()
@@ -612,10 +611,10 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self):
         init_image = init_image.resize((768, 512))
 
         model_id = "CompVis/stable-diffusion-v1-4"
-        lms = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler")
         pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
-            model_id, scheduler=lms, safety_checker=None, device_map="auto", revision="fp16", torch_dtype=torch.float16
+            model_id, safety_checker=None, device_map="auto", revision="fp16", torch_dtype=torch.float16
         )
+        pipe.set_scheduler("lms-discrete")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing(1)
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py
index 5fcdd71dd6e4..99b7d5812d62 100644
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py
@@ -359,8 +359,8 @@ def test_stable_diffusion_inpaint_pipeline_pndm(self):
         )
 
         model_id = "runwayml/stable-diffusion-inpainting"
-        pndm = PNDMScheduler.from_config(model_id, subfolder="scheduler")
-        pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None, scheduler=pndm)
+        pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None)
+        pipe.set_scheduler("pndm")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing()
@@ -396,15 +396,14 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self):
         )
 
         model_id = "runwayml/stable-diffusion-inpainting"
-        pndm = PNDMScheduler.from_config(model_id, subfolder="scheduler")
         pipe = StableDiffusionInpaintPipeline.from_pretrained(
             model_id,
             safety_checker=None,
-            scheduler=pndm,
             device_map="auto",
             revision="fp16",
             torch_dtype=torch.float16,
         )
+        pipe.set_scheduler("pndm")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing(1)
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py
index c5b2572fb79a..decfbf9abace 100644
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py
@@ -22,7 +22,6 @@
 
 from diffusers import (
     AutoencoderKL,
-    LMSDiscreteScheduler,
     PNDMScheduler,
     StableDiffusionInpaintPipeline,
     StableDiffusionInpaintPipelineLegacy,
@@ -402,12 +401,8 @@ def test_stable_diffusion_inpaint_legacy_pipeline_k_lms(self):
         )
 
         model_id = "CompVis/stable-diffusion-v1-4"
-        lms = LMSDiscreteScheduler.from_config(model_id, subfolder="scheduler")
-        pipe = StableDiffusionInpaintPipeline.from_pretrained(
-            model_id,
-            scheduler=lms,
-            safety_checker=None,
-        )
+        pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None)
+        pipe.set_scheduler("lms-discrete")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing()

From b1f573838ddc859cd8faa7f590d25d47d0ff4fd5 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 17:03:41 +0100
Subject: [PATCH 05/13] finish

---
 README.md                                      | 16 ++++++++++++++++
 docs/source/api/diffusion_pipeline.mdx         |  4 ++++
 docs/source/api/pipelines/stable_diffusion.mdx | 12 ++++++------
 docs/source/api/schedulers.mdx                 |  3 +++
 docs/source/quicktour.mdx                      | 11 ++++-------
 docs/source/using-diffusers/loading.mdx        |  3 +++
 src/diffusers/pipeline_utils.py                | 16 ++++++++++++++++
 src/diffusers/schedulers/__init__.py           |  3 +++
 8 files changed, 55 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index 64cbd15aab26..ec07b503ebc9 100644
--- a/README.md
+++ b/README.md
@@ -160,6 +160,22 @@ pipe = StableDiffusionPipeline.from_pretrained(
     torch_dtype=torch.float16,
     scheduler=lms,
 )
+```
+
+or even easier you can make use of the `set_scheduler` functionality.
+
+```python
+pipe = StableDiffusionPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", 
+    revision="fp16", 
+    torch_dtype=torch.float16,
+)
+pipe.set_scheduler("lms-discrete")
+```
+
+Then you can run the pipeline just as before.
+
+```
 pipe = pipe.to("cuda")
 
 prompt = "a photo of an astronaut riding a horse on mars"
diff --git a/docs/source/api/diffusion_pipeline.mdx b/docs/source/api/diffusion_pipeline.mdx
index b037b4e26dc1..e9fe1c126163 100644
--- a/docs/source/api/diffusion_pipeline.mdx
+++ b/docs/source/api/diffusion_pipeline.mdx
@@ -32,9 +32,13 @@ Any pipeline object can be saved locally with [`~DiffusionPipeline.save_pretrain
 [[autodoc]] DiffusionPipeline
 	- from_pretrained
 	- save_pretrained
+	- set_scheduler
 	- to
 	- device
 	- components
+	- numpy_to_pil
+	- progress_bar
+	- set_progress_bar_config
 
 ## ImagePipelineOutput
 By default diffusion pipelines return an object of class
diff --git a/docs/source/api/pipelines/stable_diffusion.mdx b/docs/source/api/pipelines/stable_diffusion.mdx
index 26d6a210adad..83ebc1b4a0ac 100644
--- a/docs/source/api/pipelines/stable_diffusion.mdx
+++ b/docs/source/api/pipelines/stable_diffusion.mdx
@@ -31,16 +31,16 @@ For more details about how Stable Diffusion works and how it differs from the ba
 
 ## Tips
 
-### How to load and use different schedulers.
+### How to use different schedulers.
 
-The stable diffusion pipeline uses [`PNDMScheduler`] scheduler by default. But `diffusers` provides many other schedulers that can be used with the stable diffusion pipeline such as [`DDIMScheduler`], [`LMSDiscreteScheduler`], [`EulerDiscreteScheduler`], [`EulerAncestralDiscreteScheduler`] etc.
-To use a different scheduler, you can pass the `scheduler` argument to `from_pretrained` method of the pipeline. For example, to use the [`EulerDiscreteScheduler`], you can do the following:
+The stable diffusion pipeline uses [`PNDMScheduler`] scheduler by default. But `diffusers` provides many other schedulers that can be used with the stable diffusion pipeline such as [`DDIMScheduler`], [`LMSDiscreteScheduler`], [`EulerDiscreteScheduler`], [`EulerAncestralDiscreteScheduler`], [`DPMSolverMultistepScheduler`], etc...
+To use a different scheduler, you can pass make use of the [`DiffusionPipeline.set_scheduler`] function to the `scheduler` of the pipeline. For example, to use the [`EulerDiscreteScheduler`], you can do the following:
 
 ```python
-from diffusers import StableDiffusionPipeline, EulerDiscreteScheduler
+from diffusers import StableDiffusionPipeline
 
-euler_scheduler = EulerDiscreteScheduler.from_config("CompVis/stable-diffusion-v1-4", subfolder="scheduler")
-pipeline = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", scheduler=euler_scheduler)
+pipeline = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
+pipeline.set_scheduler("euler-discrete")
 ```
 
 
diff --git a/docs/source/api/schedulers.mdx b/docs/source/api/schedulers.mdx
index 7ed527bedf3f..f2cc4925b382 100644
--- a/docs/source/api/schedulers.mdx
+++ b/docs/source/api/schedulers.mdx
@@ -48,6 +48,9 @@ The core API for any new scheduler must follow a limited structure.
 
 The base class [`SchedulerMixin`] implements low level utilities used by multiple schedulers.
 
+### SchedulerType
+[[autodoc]] schedulers.SchedulerType
+
 ### SchedulerMixin
 [[autodoc]] SchedulerMixin
 
diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx
index 463780a0727f..825e954700cb 100644
--- a/docs/source/quicktour.mdx
+++ b/docs/source/quicktour.mdx
@@ -115,17 +115,14 @@ Running the pipeline is then identical to the code above as it's the same model
 
 Diffusion systems can be used with multiple different [schedulers](./api/schedulers) each with their
 pros and cons. By default, Stable Diffusion runs with [`PNDMScheduler`], but it's very simple to 
-use a different scheduler. *E.g.* if you would instead like to use the [`LMSDiscreteScheduler`] scheduler,
-you could use it as follows:
+use a different scheduler. *E.g.* if you would instead like to use the [`DPMSolverMultistepScheduler`] scheduler,
+you could can just set the scheduler to `"dpm-multistep"`.
 
 ```python
 >>> from diffusers import LMSDiscreteScheduler
 
->>> scheduler = LMSDiscreteScheduler.from_config("runwayml/stable-diffusion-v1-5", subfolder="scheduler")
-
->>> generator = StableDiffusionPipeline.from_pretrained(
-...     "runwayml/stable-diffusion-v1-5", scheduler=scheduler, use_auth_token=AUTH_TOKEN
-... )
+>>> generator = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", use_auth_token=AUTH_TOKEN)
+>>> generator.set_scheduler("dpm-multistep")
 ```
 
 [Stability AI's](https://stability.ai/) Stable Diffusion model is an impressive image generation model
diff --git a/docs/source/using-diffusers/loading.mdx b/docs/source/using-diffusers/loading.mdx
index 2cb980ea618a..39a5e87c44d5 100644
--- a/docs/source/using-diffusers/loading.mdx
+++ b/docs/source/using-diffusers/loading.mdx
@@ -379,6 +379,9 @@ dpm = DPMSolverMultistepScheduler.from_config(repo_id, subfolder="scheduler")
 pipeline = StableDiffusionPipeline.from_pretrained(repo_id, scheduler=dpm)
 ```
 
+**Note**: If you are often changing schedulers within the same script it is recommended to make use 
+of [`DiffusionPipeline.set_scheduler`] instead.
+
 ## API
 
 [[autodoc]] modeling_utils.ModelMixin
diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index b1bda4381fcc..b5f00df960f2 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -711,7 +711,23 @@ def progress_bar(self, iterable):
         return tqdm(iterable, **self._progress_bar_config)
 
     def set_scheduler(self, scheduler_type=Union[str, Dict[str, str]]):
+        r"""
+
+        Parameters:
+            scheduler_type (`str` or `Dict[str, str]`):
+                Can be either a string representing the type the scheduler should be set to or a mapping component name
+                to scheduler types in case the pipeline has multiple schedulers. Make sure to set the schedulers to one
+                of the officially supported scheduler types of [`schedulers.SchedulerType`].
+
+        Examples:
+
+        >>> from diffusers import DiffusionPipeline
+
+        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") >>>
+        pipe.set_scheduler("euler-discrete")
+        """
         schedulers = {k: type(v) for k, v in self.components.items() if isinstance(v, SchedulerMixin)}
+
         if isinstance(scheduler_type, str) and len(set(schedulers.values())) > 1:
             raise ValueError(
                 f"The pipeline {self} contains the schedulers {schedulers}. Please make sure to provide a dictionary"
diff --git a/src/diffusers/schedulers/__init__.py b/src/diffusers/schedulers/__init__.py
index 5e43ce20a2c5..4447a89daff6 100644
--- a/src/diffusers/schedulers/__init__.py
+++ b/src/diffusers/schedulers/__init__.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from collections import OrderedDict
+from enum import Enum
 
 from ..utils import is_flax_available, is_scipy_available, is_torch_available
 
@@ -70,3 +71,5 @@
     ]
 )
 CLASS_TO_SCHEDULER_TYPE_MAPPING = OrderedDict({v: k for k, v in SCHEDULER_TYPE_TO_CLASS_MAPPING.items()})
+
+SchedulerType = Enum("SchedulerType", SCHEDULER_TYPE_TO_CLASS_MAPPING.keys())

From 5f1cf97a775f6b106650ccfd0fdff24eeccad440 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 17:11:52 +0100
Subject: [PATCH 06/13] finish

---
 src/diffusers/pipeline_utils.py      | 6 +++---
 src/diffusers/schedulers/__init__.py | 7 ++++++-
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index b5f00df960f2..1f8b244fe341 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -34,7 +34,7 @@
 from .dynamic_modules_utils import get_class_from_dynamic_module
 from .hub_utils import http_user_agent
 from .modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT
-from .schedulers import CLASS_TO_SCHEDULER_TYPE_MAPPING, SCHEDULER_TYPE_TO_CLASS_MAPPING
+from .schedulers import CLASS_TO_SCHEDULER_TYPE_MAPPING, SCHEDULER_TYPE_TO_CLASS_MAPPING, SchedulerType
 from .schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME, SchedulerMixin
 from .utils import (
     CONFIG_NAME,
@@ -710,7 +710,7 @@ def progress_bar(self, iterable):
 
         return tqdm(iterable, **self._progress_bar_config)
 
-    def set_scheduler(self, scheduler_type=Union[str, Dict[str, str]]):
+    def set_scheduler(self, scheduler_type=Union[SchedulerType, str, Dict[str, str]]):
         r"""
 
         Parameters:
@@ -723,7 +723,7 @@ def set_scheduler(self, scheduler_type=Union[str, Dict[str, str]]):
 
         >>> from diffusers import DiffusionPipeline
 
-        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") >>>
+        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") >>> >>>
         pipe.set_scheduler("euler-discrete")
         """
         schedulers = {k: type(v) for k, v in self.components.items() if isinstance(v, SchedulerMixin)}
diff --git a/src/diffusers/schedulers/__init__.py b/src/diffusers/schedulers/__init__.py
index 4447a89daff6..b2ebcee416b0 100644
--- a/src/diffusers/schedulers/__init__.py
+++ b/src/diffusers/schedulers/__init__.py
@@ -72,4 +72,9 @@
 )
 CLASS_TO_SCHEDULER_TYPE_MAPPING = OrderedDict({v: k for k, v in SCHEDULER_TYPE_TO_CLASS_MAPPING.items()})
 
-SchedulerType = Enum("SchedulerType", SCHEDULER_TYPE_TO_CLASS_MAPPING.keys())
+SchedulerType = Enum("SchedulerType", list(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys()))
+SchedulerType.__doc__ = (
+    """Possible values for the `scheduler_type` argument in [`DiffusionPipeline.set_scheduler`]. Useful for tab-completion in
+an IDE. Possible values are"""
+    + " ,".join(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys())
+)

From 1a5501b7e9b910026eff220f9b951de717cc877d Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 17:13:56 +0100
Subject: [PATCH 07/13] finish all

---
 src/diffusers/schedulers/__init__.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/diffusers/schedulers/__init__.py b/src/diffusers/schedulers/__init__.py
index b2ebcee416b0..c578a93f5936 100644
--- a/src/diffusers/schedulers/__init__.py
+++ b/src/diffusers/schedulers/__init__.py
@@ -73,8 +73,7 @@
 CLASS_TO_SCHEDULER_TYPE_MAPPING = OrderedDict({v: k for k, v in SCHEDULER_TYPE_TO_CLASS_MAPPING.items()})
 
 SchedulerType = Enum("SchedulerType", list(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys()))
-SchedulerType.__doc__ = (
-    """Possible values for the `scheduler_type` argument in [`DiffusionPipeline.set_scheduler`]. Useful for tab-completion in
-an IDE. Possible values are"""
-    + " ,".join(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys())
+SchedulerType.__doc__ = """Possible values for the `scheduler_type` argument in [`DiffusionPipeline.set_scheduler`]. Useful for tab-completion in
+an IDE. Possible values are""" + "\n- ".join(
+    SCHEDULER_TYPE_TO_CLASS_MAPPING.keys()
 )

From d42db38329b4c4fcc4962896c753d8d423c77a2e Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 18:59:33 +0100
Subject: [PATCH 08/13] finish

---
 src/diffusers/pipeline_utils.py      | 15 ++++++++++-----
 src/diffusers/schedulers/__init__.py | 18 +++++++++---------
 tests/test_pipelines.py              | 28 ++++++++++++++++------------
 3 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index 1f8b244fe341..448189576920 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -710,7 +710,7 @@ def progress_bar(self, iterable):
 
         return tqdm(iterable, **self._progress_bar_config)
 
-    def set_scheduler(self, scheduler_type=Union[SchedulerType, str, Dict[str, str]]):
+    def set_scheduler(self, scheduler_type=Union[str, SchedulerType, Dict[str, str], Dict[str, SchedulerType]]):
         r"""
 
         Parameters:
@@ -721,18 +721,20 @@ def set_scheduler(self, scheduler_type=Union[SchedulerType, str, Dict[str, str]]
 
         Examples:
 
+        ```py
         >>> from diffusers import DiffusionPipeline
 
-        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") >>> >>>
-        pipe.set_scheduler("euler-discrete")
+        >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
+        >>> pipe.set_scheduler("euler-discrete")
+        ```
         """
         schedulers = {k: type(v) for k, v in self.components.items() if isinstance(v, SchedulerMixin)}
 
         if isinstance(scheduler_type, str) and len(set(schedulers.values())) > 1:
             raise ValueError(
                 f"The pipeline {self} contains the schedulers {schedulers}. Please make sure to provide a dictionary"
-                f" that maps the componet names {schedulers.keys()} to scheduler types instead of just one scheduler"
-                f" type {scheduler_type}"
+                f" that maps the componet names {schedulers.keys()} to scheduler types. Providing just one scheduler"
+                f" type {scheduler_type} is ambiguous."
             )
         elif isinstance(scheduler_type, dict):
             is_type_scheduler = {k: k in schedulers for k in scheduler_type.keys()}
@@ -748,6 +750,9 @@ def set_scheduler(self, scheduler_type=Union[SchedulerType, str, Dict[str, str]]
         )
 
         for component_name, scheduler_type in scheduler_mapping.items():
+            if isinstance(scheduler_type, SchedulerType):
+                scheduler_type = scheduler_type.name
+
             scheduler_class = SCHEDULER_TYPE_TO_CLASS_MAPPING.get(scheduler_type, None)
             current_scheduler = getattr(self, component_name)
 
diff --git a/src/diffusers/schedulers/__init__.py b/src/diffusers/schedulers/__init__.py
index c578a93f5936..d8dd906e6f33 100644
--- a/src/diffusers/schedulers/__init__.py
+++ b/src/diffusers/schedulers/__init__.py
@@ -57,23 +57,23 @@
     [
         ("ddim", DDIMScheduler),
         ("ddpm", DDPMScheduler),
-        ("dpm-multistep", DPMSolverMultistepScheduler),
-        ("euler-ancestral-discrete", EulerAncestralDiscreteScheduler),
-        ("euler-discrete", EulerDiscreteScheduler),
+        ("dpm_multistep", DPMSolverMultistepScheduler),
+        ("euler_ancestral_discrete", EulerAncestralDiscreteScheduler),
+        ("euler_discrete", EulerDiscreteScheduler),
         ("ipndm", IPNDMScheduler),
-        ("karras-ve", KarrasVeScheduler),
+        ("karras_ve", KarrasVeScheduler),
         ("pndm", PNDMScheduler),
         ("repaint", RePaintScheduler),
-        ("score-sde-ve", ScoreSdeVeScheduler),
-        ("score-sde-vp", ScoreSdeVpScheduler),
-        ("vq-diffusion", VQDiffusionScheduler),
-        ("lms-discrete", LMSDiscreteScheduler),
+        ("score_sde_ve", ScoreSdeVeScheduler),
+        ("score_sde_vp", ScoreSdeVpScheduler),
+        ("vq_diffusion", VQDiffusionScheduler),
+        ("lms_discrete", LMSDiscreteScheduler),
     ]
 )
 CLASS_TO_SCHEDULER_TYPE_MAPPING = OrderedDict({v: k for k, v in SCHEDULER_TYPE_TO_CLASS_MAPPING.items()})
 
 SchedulerType = Enum("SchedulerType", list(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys()))
 SchedulerType.__doc__ = """Possible values for the `scheduler_type` argument in [`DiffusionPipeline.set_scheduler`]. Useful for tab-completion in
-an IDE. Possible values are""" + "\n- ".join(
+an IDE. Possible values are\n""" + "\n- ".join(
     SCHEDULER_TYPE_TO_CLASS_MAPPING.keys()
 )
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 55f1609901cd..3b860920a379 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -44,6 +44,7 @@
     logging,
 )
 from diffusers.pipeline_utils import DiffusionPipeline
+from diffusers.schedulers import SchedulerType
 from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME
 from diffusers.utils import CONFIG_NAME, WEIGHTS_NAME, floats_tensor, slow, torch_device
 from diffusers.utils.testing_utils import CaptureLogger, get_tests_dir, require_torch_gpu
@@ -426,28 +427,31 @@ def test_set_scheduler(self):
         assert isinstance(sd.scheduler, DDPMScheduler)
         sd.set_scheduler("pndm")
         assert isinstance(sd.scheduler, PNDMScheduler)
-        sd.set_scheduler("lms-discrete")
+        sd.set_scheduler("lms_discrete")
         assert isinstance(sd.scheduler, LMSDiscreteScheduler)
-        sd.set_scheduler("euler-discrete")
+        sd.set_scheduler("euler_discrete")
         assert isinstance(sd.scheduler, EulerDiscreteScheduler)
-        sd.set_scheduler("euler-ancestral-discrete")
+        sd.set_scheduler("euler_ancestral_discrete")
         assert isinstance(sd.scheduler, EulerAncestralDiscreteScheduler)
-        sd.set_scheduler("dpm-multistep")
+        sd.set_scheduler("dpm_multistep")
         assert isinstance(sd.scheduler, DPMSolverMultistepScheduler)
-
-        sd.set_scheduler({"scheduler": "dpm-multistep"})
+        sd.set_scheduler(SchedulerType.dpm_multistep)
+        assert isinstance(sd.scheduler, DPMSolverMultistepScheduler)
+        sd.set_scheduler("dpm_multistep")
+        assert isinstance(sd.scheduler, DPMSolverMultistepScheduler)
+        sd.set_scheduler({"scheduler": "dpm_multistep"})
         assert isinstance(sd.scheduler, DPMSolverMultistepScheduler)
 
         logger = logging.get_logger("diffusers.pipeline_utils")
         with self.assertRaises(ValueError) as error_1:
-            sd.set_scheduler({"schedule": "dpm-multistep"})
+            sd.set_scheduler({"schedule": "dpm_multistep"})
 
         with self.assertRaises(ValueError) as error_2:
-            sd.set_scheduler({"scheduler": "dpm-multiste"})
+            sd.set_scheduler({"scheduler": "dpm_multiste"})
 
         logger.setLevel(diffusers.logging.INFO)
         with CaptureLogger(logger) as cap_logger:
-            sd.set_scheduler({"scheduler": "dpm-multistep"})
+            sd.set_scheduler({"scheduler": "dpm_multistep"})
 
         with CaptureLogger(logger) as cap_logger_warn:
             sd.set_scheduler({"scheduler": "ipndm"})
@@ -457,10 +461,10 @@ def test_set_scheduler(self):
             == "The following component names are not schedulers ['schedule']. Please make sure to only set new"
             " scheduler types for dict_keys(['scheduler'])."
         )
-        assert "dpm-multiste does not exist, make sure to chose a scheduler type from" in str(error_2.exception)
-        assert cap_logger.out == "Changing scheduler from type dpm-multistep to dpm-multistep.\n"
+        assert "dpm_multiste does not exist, make sure to chose a scheduler type from" in str(error_2.exception)
+        assert cap_logger.out == "Changing scheduler from type dpm_multistep to dpm_multistep.\n"
         assert (
-            "Changing scheduler from type dpm-multistep to an uncompatible scheduler type ipndm."
+            "Changing scheduler from type dpm_multistep to an uncompatible scheduler type ipndm."
             in cap_logger_warn.out
         )
 

From b38f03bf2fb296b684dab2ac93940594a8294dca Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 19:01:20 +0100
Subject: [PATCH 09/13] finish

---
 README.md                                                     | 2 +-
 docs/source/api/pipelines/stable_diffusion.mdx                | 2 +-
 src/diffusers/pipeline_utils.py                               | 2 +-
 tests/pipelines/stable_diffusion/test_stable_diffusion.py     | 2 +-
 .../stable_diffusion/test_stable_diffusion_img2img.py         | 4 ++--
 .../stable_diffusion/test_stable_diffusion_inpaint_legacy.py  | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index ec07b503ebc9..8c6fd86326ac 100644
--- a/README.md
+++ b/README.md
@@ -170,7 +170,7 @@ pipe = StableDiffusionPipeline.from_pretrained(
     revision="fp16", 
     torch_dtype=torch.float16,
 )
-pipe.set_scheduler("lms-discrete")
+pipe.set_scheduler("lms_discrete")
 ```
 
 Then you can run the pipeline just as before.
diff --git a/docs/source/api/pipelines/stable_diffusion.mdx b/docs/source/api/pipelines/stable_diffusion.mdx
index 83ebc1b4a0ac..40226eb55520 100644
--- a/docs/source/api/pipelines/stable_diffusion.mdx
+++ b/docs/source/api/pipelines/stable_diffusion.mdx
@@ -40,7 +40,7 @@ To use a different scheduler, you can pass make use of the [`DiffusionPipeline.s
 from diffusers import StableDiffusionPipeline
 
 pipeline = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4")
-pipeline.set_scheduler("euler-discrete")
+pipeline.set_scheduler("euler_discrete")
 ```
 
 
diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index 448189576920..9e0ca5aacd3d 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -725,7 +725,7 @@ def set_scheduler(self, scheduler_type=Union[str, SchedulerType, Dict[str, str],
         >>> from diffusers import DiffusionPipeline
 
         >>> pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5")
-        >>> pipe.set_scheduler("euler-discrete")
+        >>> pipe.set_scheduler("euler_discrete")
         ```
         """
         schedulers = {k: type(v) for k, v in self.components.items() if isinstance(v, SchedulerMixin)}
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/tests/pipelines/stable_diffusion/test_stable_diffusion.py
index 2e085cc28d03..1dfa7cfad3ba 100644
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion.py
@@ -673,7 +673,7 @@ def test_lms_stable_diffusion_pipeline(self):
         model_id = "CompVis/stable-diffusion-v1-1"
         pipe = StableDiffusionPipeline.from_pretrained(model_id).to(torch_device)
         pipe.set_progress_bar_config(disable=None)
-        pipe.set_scheduler("lms-discrete")
+        pipe.set_scheduler("lms_discrete")
 
         prompt = "a photograph of an astronaut riding a horse"
         generator = torch.Generator(device=torch_device).manual_seed(0)
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py
index a4c9b3b74ae3..a1ab8a7db58c 100644
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py
@@ -523,7 +523,7 @@ def test_stable_diffusion_img2img_pipeline_k_lms(self):
             model_id,
             safety_checker=None,
         )
-        pipe.set_scheduler("lms-discrete")
+        pipe.set_scheduler("lms_discrete")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing()
@@ -614,7 +614,7 @@ def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self):
         pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
             model_id, safety_checker=None, device_map="auto", revision="fp16", torch_dtype=torch.float16
         )
-        pipe.set_scheduler("lms-discrete")
+        pipe.set_scheduler("lms_discrete")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing(1)
diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py
index decfbf9abace..aed5931cf397 100644
--- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py
+++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint_legacy.py
@@ -402,7 +402,7 @@ def test_stable_diffusion_inpaint_legacy_pipeline_k_lms(self):
 
         model_id = "CompVis/stable-diffusion-v1-4"
         pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None)
-        pipe.set_scheduler("lms-discrete")
+        pipe.set_scheduler("lms_discrete")
         pipe.to(torch_device)
         pipe.set_progress_bar_config(disable=None)
         pipe.enable_attention_slicing()

From 04451c9d913c30b69e2337c733f1f26a99db98db Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 19:05:12 +0100
Subject: [PATCH 10/13] up

---
 src/diffusers/schedulers/__init__.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/diffusers/schedulers/__init__.py b/src/diffusers/schedulers/__init__.py
index d8dd906e6f33..c773f2552003 100644
--- a/src/diffusers/schedulers/__init__.py
+++ b/src/diffusers/schedulers/__init__.py
@@ -73,7 +73,9 @@
 CLASS_TO_SCHEDULER_TYPE_MAPPING = OrderedDict({v: k for k, v in SCHEDULER_TYPE_TO_CLASS_MAPPING.items()})
 
 SchedulerType = Enum("SchedulerType", list(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys()))
-SchedulerType.__doc__ = """Possible values for the `scheduler_type` argument in [`DiffusionPipeline.set_scheduler`]. Useful for tab-completion in
-an IDE. Possible values are\n""" + "\n- ".join(
-    SCHEDULER_TYPE_TO_CLASS_MAPPING.keys()
+SchedulerType.__doc__ = (
+    """Possible values for the `scheduler_type` argument in [`DiffusionPipeline.set_scheduler`]. Useful for tab-completion in
+an IDE. Possible values are"""
+    + "\n"
+    + "\n- ".join(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys())
 )

From eb083b800384acabad1b73dcf210615e99eef7d0 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 23:13:44 +0100
Subject: [PATCH 11/13] finalize

---
 .../cd4a85374821636f69971f3e1840fc21546339a4  | 101 ++++++++++++++++++
 .../refs/main                                 |   1 +
 .../pipeline.py                               |   1 +
 src/diffusers/configuration_utils.py          |  31 +++++-
 src/diffusers/models/vae.py                   |   1 +
 src/diffusers/pipeline_utils.py               |  17 +--
 tests/test_pipelines.py                       |  42 ++++++++
 7 files changed, 184 insertions(+), 10 deletions(-)
 create mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
 create mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
 create mode 120000 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py

diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4 b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
new file mode 100644
index 000000000000..cd4a85374821
--- /dev/null
+++ b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
@@ -0,0 +1,101 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+
+# limitations under the License.
+
+
+from typing import Optional, Tuple, Union
+
+import torch
+
+from diffusers.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
+
+
+class CustomPipeline(DiffusionPipeline):
+    r"""
+    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
+    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
+
+    Parameters:
+        unet ([`UNet2DModel`]): U-Net architecture to denoise the encoded image.
+        scheduler ([`SchedulerMixin`]):
+            A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of
+            [`DDPMScheduler`], or [`DDIMScheduler`].
+    """
+
+    def __init__(self, unet, scheduler):
+        super().__init__()
+        self.register_modules(unet=unet, scheduler=scheduler)
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        batch_size: int = 1,
+        generator: Optional[torch.Generator] = None,
+        num_inference_steps: int = 50,
+        output_type: Optional[str] = "pil",
+        return_dict: bool = True,
+        **kwargs,
+    ) -> Union[ImagePipelineOutput, Tuple]:
+        r"""
+        Args:
+            batch_size (`int`, *optional*, defaults to 1):
+                The number of images to generate.
+            generator (`torch.Generator`, *optional*):
+                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
+                deterministic.
+            eta (`float`, *optional*, defaults to 0.0):
+                The eta parameter which controls the scale of the variance (0 is DDIM and 1 is one type of DDPM).
+            num_inference_steps (`int`, *optional*, defaults to 50):
+                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+                expense of slower inference.
+            output_type (`str`, *optional*, defaults to `"pil"`):
+                The output format of the generate image. Choose between
+                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+            return_dict (`bool`, *optional*, defaults to `True`):
+                Whether or not to return a [`~pipeline_utils.ImagePipelineOutput`] instead of a plain tuple.
+
+        Returns:
+            [`~pipeline_utils.ImagePipelineOutput`] or `tuple`: [`~pipelines.utils.ImagePipelineOutput`] if
+            `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is a list with the
+            generated images.
+        """
+
+        # Sample gaussian noise to begin loop
+        image = torch.randn(
+            (batch_size, self.unet.in_channels, self.unet.sample_size, self.unet.sample_size),
+            generator=generator,
+        )
+        image = image.to(self.device)
+
+        # set step values
+        self.scheduler.set_timesteps(num_inference_steps)
+
+        for t in self.progress_bar(self.scheduler.timesteps):
+            # 1. predict noise model_output
+            model_output = self.unet(image, t).sample
+
+            # 2. predict previous mean of image x_t-1 and add variance depending on eta
+            # eta corresponds to η in paper and should be between [0, 1]
+            # do x_t -> x_t-1
+            image = self.scheduler.step(model_output, t, image).prev_sample
+
+        image = (image / 2 + 0.5).clamp(0, 1)
+        image = image.cpu().permute(0, 2, 3, 1).numpy()
+        if output_type == "pil":
+            image = self.numpy_to_pil(image)
+
+        if not return_dict:
+            return (image,)
+
+        return ImagePipelineOutput(images=image), "This is a test"
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
new file mode 100644
index 000000000000..b8b710da3d11
--- /dev/null
+++ b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
@@ -0,0 +1 @@
+59d7c3f29a8379f8d9d53e3e111212a3da855e70
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
new file mode 120000
index 000000000000..3bf3ab2bdf65
--- /dev/null
+++ b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
@@ -0,0 +1 @@
+../../blobs/cd4a85374821636f69971f3e1840fc21546339a4
\ No newline at end of file
diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py
index fc6ac9b5b97e..a370bee8f74f 100644
--- a/src/diffusers/configuration_utils.py
+++ b/src/diffusers/configuration_utils.py
@@ -63,7 +63,6 @@ def register_to_config(self, **kwargs):
         kwargs["_class_name"] = self.__class__.__name__
         kwargs["_diffusers_version"] = __version__
 
-        # Special case for `kwargs` used in deprecation warning added to schedulers
         # TODO: remove this when we remove the deprecation warning, and the `kwargs` argument,
         # or solve in a more general way.
         kwargs.pop("kwargs", None)
@@ -83,6 +82,16 @@ def register_to_config(self, **kwargs):
 
         self._internal_dict = FrozenDict(internal_dict)
 
+    def register_to_hidden_config(self, **kwargs):
+        if not hasattr(self, "_hidden_internal_dict"):
+            internal_hidden_dict = kwargs
+        else:
+            prev_hidden_dict = dict(self._hidden_internal_dict)
+            internal_hidden_dict = {**self._hidden_internal_dict, **kwargs}
+            logger.debug(f"Updating config from {prev_hidden_dict} to {internal_hidden_dict}")
+
+        self._hidden_internal_dict = FrozenDict(internal_hidden_dict)
+
     def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):
         """
         Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the
@@ -164,7 +173,7 @@ def from_config(cls, pretrained_model_name_or_path: Union[str, os.PathLike], ret
 
         """
         config_dict = cls.get_config_dict(pretrained_model_name_or_path=pretrained_model_name_or_path, **kwargs)
-        init_dict, unused_kwargs = cls.extract_init_dict(config_dict, **kwargs)
+        init_dict, unused_kwargs, hidden_dict = cls.extract_init_dict(config_dict, **kwargs)
 
         # Allow dtype to be specified on initialization
         if "dtype" in unused_kwargs:
@@ -172,6 +181,10 @@ def from_config(cls, pretrained_model_name_or_path: Union[str, os.PathLike], ret
 
         # Return model and optionally state and/or unused_kwargs
         model = cls(**init_dict)
+
+        # make sure to also save config parameters that might be used for compatible classes
+        model.register_to_hidden_config(**hidden_dict)
+
         return_tuple = (model,)
 
         # Flax schedulers have a state, so return it.
@@ -291,6 +304,9 @@ def _get_init_keys(cls):
 
     @classmethod
     def extract_init_dict(cls, config_dict, **kwargs):
+        # 0. Copy origin config dict
+        original_dict = {k: v for k, v in config_dict.items()}
+
         # 1. Retrieve expected config attributes from __init__ signature
         expected_keys = cls._get_init_keys(cls)
         expected_keys.remove("self")
@@ -364,7 +380,10 @@ def extract_init_dict(cls, config_dict, **kwargs):
         # 6. Define unused keyword arguments
         unused_kwargs = {**config_dict, **kwargs}
 
-        return init_dict, unused_kwargs
+        # 7. Define "hidden" config parameters that were saved for compatible classes
+        hidden_config_dict = {k: v for k, v in original_dict.items() if k not in init_dict and not k.startswith("_")}
+
+        return init_dict, unused_kwargs, hidden_config_dict
 
     @classmethod
     def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]):
@@ -379,6 +398,12 @@ def __repr__(self):
     def config(self) -> Dict[str, Any]:
         return self._internal_dict
 
+    @property
+    def hidden_config(self) -> Dict[str, Any]:
+        if not hasattr(self, "_hidden_internal_dict"):
+            return {}
+        return self._hidden_internal_dict
+
     def to_json_string(self) -> str:
         """
         Serializes this instance to a JSON string.
diff --git a/src/diffusers/models/vae.py b/src/diffusers/models/vae.py
index 30de343d08ee..dda83f29445e 100644
--- a/src/diffusers/models/vae.py
+++ b/src/diffusers/models/vae.py
@@ -84,6 +84,7 @@ def __init__(
         self.mid_block = None
         self.down_blocks = nn.ModuleList([])
 
+        # import ipdb; ipdb.set_trace()
         # down
         output_channel = block_out_channels[0]
         for i, down_block_type in enumerate(down_block_types):
diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index 9e0ca5aacd3d..7ea91a814fa7 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -208,7 +208,7 @@ def to(self, torch_device: Optional[Union[str, torch.device]] = None):
         if torch_device is None:
             return self
 
-        module_names, _ = self.extract_init_dict(dict(self.config))
+        module_names, _, _ = self.extract_init_dict(dict(self.config))
         for name in module_names.keys():
             module = getattr(self, name)
             if isinstance(module, torch.nn.Module):
@@ -229,7 +229,7 @@ def device(self) -> torch.device:
         Returns:
             `torch.device`: The torch device on which the pipeline is located.
         """
-        module_names, _ = self.extract_init_dict(dict(self.config))
+        module_names, _, _ = self.extract_init_dict(dict(self.config))
         for name in module_names.keys():
             module = getattr(self, name)
             if isinstance(module, torch.nn.Module):
@@ -514,7 +514,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.P
         expected_modules = set(inspect.signature(pipeline_class.__init__).parameters.keys()) - set(["self"])
         passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs}
 
-        init_dict, unused_kwargs = pipeline_class.extract_init_dict(config_dict, **kwargs)
+        init_dict, unused_kwargs, _ = pipeline_class.extract_init_dict(config_dict, **kwargs)
 
         if len(unused_kwargs) > 0:
             logger.warning(f"Keyword arguments {unused_kwargs} not recognized.")
@@ -762,7 +762,7 @@ def set_scheduler(self, scheduler_type=Union[str, SchedulerType, Dict[str, str],
                     f" {', '.join(SCHEDULER_TYPE_TO_CLASS_MAPPING.keys())}."
                 )
 
-            if scheduler_class not in current_scheduler._compatible_classes and scheduler_class != type(
+            if scheduler_class.__name__ not in current_scheduler._compatible_classes and scheduler_class != type(
                 current_scheduler
             ):
                 diffusers_library = importlib.import_module(__name__.split(".")[0])
@@ -774,13 +774,16 @@ def set_scheduler(self, scheduler_type=Union[str, SchedulerType, Dict[str, str],
                     f"Changing scheduler from type {CLASS_TO_SCHEDULER_TYPE_MAPPING[type(current_scheduler)]} to an"
                     f" uncompatible scheduler type {scheduler_type}. This is very likely going to lead to incorrect"
                     f" predictions when running the pipeline. Make sure to set {component_name} to a scheduler of type"
-                    f" {[' ,'.join(_compatible_class_types)]}."
+                    f" {[', '.join(_compatible_class_types)]}."
                 )
 
-            scheduler_config = current_scheduler.config
-            scheduler_init_dict, _ = scheduler_class.extract_init_dict(scheduler_config)
+            # new scheduler config is current config + hidden config
+            scheduler_config = {**current_scheduler.config, **current_scheduler.hidden_config}
+
+            scheduler_init_dict, _, hidden_dict = scheduler_class.extract_init_dict(scheduler_config)
 
             scheduler = scheduler_class(**scheduler_init_dict)
+            scheduler.register_to_hidden_config(**hidden_dict)
 
             logger.info(
                 f"Changing scheduler from type {CLASS_TO_SCHEDULER_TYPE_MAPPING[type(current_scheduler)]} to"
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index 3b860920a379..c47ba9861499 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -468,6 +468,48 @@ def test_set_scheduler(self):
             in cap_logger_warn.out
         )
 
+    def test_set_scheduler_consitency(self):
+        unet = self.dummy_cond_unet
+        pndm = PNDMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler")
+        ddim = DDIMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler")
+        vae = self.dummy_vae
+        bert = self.dummy_text_encoder
+        tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip")
+
+        sd = StableDiffusionPipeline(
+            unet=unet,
+            scheduler=pndm,
+            vae=vae,
+            text_encoder=bert,
+            tokenizer=tokenizer,
+            safety_checker=None,
+            feature_extractor=self.dummy_extractor,
+        )
+
+        pndm_config = sd.scheduler.config
+        sd.set_scheduler("ddpm")
+        sd.set_scheduler("pndm")
+        pndm_config_2 = sd.scheduler.config
+
+        assert dict(pndm_config) == dict(pndm_config_2)
+
+        sd = StableDiffusionPipeline(
+            unet=unet,
+            scheduler=ddim,
+            vae=vae,
+            text_encoder=bert,
+            tokenizer=tokenizer,
+            safety_checker=None,
+            feature_extractor=self.dummy_extractor,
+        )
+
+        ddim_config = sd.scheduler.config
+        sd.set_scheduler("lms_discrete")
+        sd.set_scheduler("ddim")
+        ddim_config_2 = sd.scheduler.config
+
+        assert dict(ddim_config) == dict(ddim_config_2)
+
 
 @slow
 class PipelineSlowTests(unittest.TestCase):

From dfd47ce65eecb932873d3767ba003fa78ae93e42 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sat, 12 Nov 2022 23:13:54 +0100
Subject: [PATCH 12/13] up

---
 .../cd4a85374821636f69971f3e1840fc21546339a4  | 101 ------------------
 .../refs/main                                 |   1 -
 .../pipeline.py                               |   1 -
 3 files changed, 103 deletions(-)
 delete mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
 delete mode 100644 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
 delete mode 120000 hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py

diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4 b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
deleted file mode 100644
index cd4a85374821..000000000000
--- a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/blobs/cd4a85374821636f69971f3e1840fc21546339a4
+++ /dev/null
@@ -1,101 +0,0 @@
-# Copyright 2022 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-
-# limitations under the License.
-
-
-from typing import Optional, Tuple, Union
-
-import torch
-
-from diffusers.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
-
-
-class CustomPipeline(DiffusionPipeline):
-    r"""
-    This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
-    library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
-
-    Parameters:
-        unet ([`UNet2DModel`]): U-Net architecture to denoise the encoded image.
-        scheduler ([`SchedulerMixin`]):
-            A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of
-            [`DDPMScheduler`], or [`DDIMScheduler`].
-    """
-
-    def __init__(self, unet, scheduler):
-        super().__init__()
-        self.register_modules(unet=unet, scheduler=scheduler)
-
-    @torch.no_grad()
-    def __call__(
-        self,
-        batch_size: int = 1,
-        generator: Optional[torch.Generator] = None,
-        num_inference_steps: int = 50,
-        output_type: Optional[str] = "pil",
-        return_dict: bool = True,
-        **kwargs,
-    ) -> Union[ImagePipelineOutput, Tuple]:
-        r"""
-        Args:
-            batch_size (`int`, *optional*, defaults to 1):
-                The number of images to generate.
-            generator (`torch.Generator`, *optional*):
-                A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation
-                deterministic.
-            eta (`float`, *optional*, defaults to 0.0):
-                The eta parameter which controls the scale of the variance (0 is DDIM and 1 is one type of DDPM).
-            num_inference_steps (`int`, *optional*, defaults to 50):
-                The number of denoising steps. More denoising steps usually lead to a higher quality image at the
-                expense of slower inference.
-            output_type (`str`, *optional*, defaults to `"pil"`):
-                The output format of the generate image. Choose between
-                [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
-            return_dict (`bool`, *optional*, defaults to `True`):
-                Whether or not to return a [`~pipeline_utils.ImagePipelineOutput`] instead of a plain tuple.
-
-        Returns:
-            [`~pipeline_utils.ImagePipelineOutput`] or `tuple`: [`~pipelines.utils.ImagePipelineOutput`] if
-            `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is a list with the
-            generated images.
-        """
-
-        # Sample gaussian noise to begin loop
-        image = torch.randn(
-            (batch_size, self.unet.in_channels, self.unet.sample_size, self.unet.sample_size),
-            generator=generator,
-        )
-        image = image.to(self.device)
-
-        # set step values
-        self.scheduler.set_timesteps(num_inference_steps)
-
-        for t in self.progress_bar(self.scheduler.timesteps):
-            # 1. predict noise model_output
-            model_output = self.unet(image, t).sample
-
-            # 2. predict previous mean of image x_t-1 and add variance depending on eta
-            # eta corresponds to η in paper and should be between [0, 1]
-            # do x_t -> x_t-1
-            image = self.scheduler.step(model_output, t, image).prev_sample
-
-        image = (image / 2 + 0.5).clamp(0, 1)
-        image = image.cpu().permute(0, 2, 3, 1).numpy()
-        if output_type == "pil":
-            image = self.numpy_to_pil(image)
-
-        if not return_dict:
-            return (image,)
-
-        return ImagePipelineOutput(images=image), "This is a test"
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
deleted file mode 100644
index b8b710da3d11..000000000000
--- a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/refs/main
+++ /dev/null
@@ -1 +0,0 @@
-59d7c3f29a8379f8d9d53e3e111212a3da855e70
\ No newline at end of file
diff --git a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py b/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
deleted file mode 120000
index 3bf3ab2bdf65..000000000000
--- a/hf-internal-testing/diffusers-dummy-pipeline/models--hf-internal-testing--diffusers-dummy-pipeline/snapshots/59d7c3f29a8379f8d9d53e3e111212a3da855e70/pipeline.py
+++ /dev/null
@@ -1 +0,0 @@
-../../blobs/cd4a85374821636f69971f3e1840fc21546339a4
\ No newline at end of file

From adb93277fda4027b316eca3bd30be10428c54018 Mon Sep 17 00:00:00 2001
From: Patrick von Platen <patrick.v.platen@gmail.com>
Date: Sun, 13 Nov 2022 00:28:38 +0100
Subject: [PATCH 13/13] up

---
 src/diffusers/configuration_utils.py | 31 ++++++++++------------------
 src/diffusers/pipeline_utils.py      |  8 +------
 tests/test_pipelines.py              |  6 ++++--
 3 files changed, 16 insertions(+), 29 deletions(-)

diff --git a/src/diffusers/configuration_utils.py b/src/diffusers/configuration_utils.py
index a370bee8f74f..29ee0e8aa7a7 100644
--- a/src/diffusers/configuration_utils.py
+++ b/src/diffusers/configuration_utils.py
@@ -82,16 +82,6 @@ def register_to_config(self, **kwargs):
 
         self._internal_dict = FrozenDict(internal_dict)
 
-    def register_to_hidden_config(self, **kwargs):
-        if not hasattr(self, "_hidden_internal_dict"):
-            internal_hidden_dict = kwargs
-        else:
-            prev_hidden_dict = dict(self._hidden_internal_dict)
-            internal_hidden_dict = {**self._hidden_internal_dict, **kwargs}
-            logger.debug(f"Updating config from {prev_hidden_dict} to {internal_hidden_dict}")
-
-        self._hidden_internal_dict = FrozenDict(internal_hidden_dict)
-
     def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs):
         """
         Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the
@@ -113,7 +103,9 @@ def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool
         logger.info(f"Configuration saved in {output_config_file}")
 
     @classmethod
-    def from_config(cls, pretrained_model_name_or_path: Union[str, os.PathLike], return_unused_kwargs=False, **kwargs):
+    def from_config(
+        cls, pretrained_model_name_or_path: Union[str, os.PathLike, dict], return_unused_kwargs=False, **kwargs
+    ):
         r"""
         Instantiate a Python class from a pre-defined JSON-file.
 
@@ -172,7 +164,11 @@ def from_config(cls, pretrained_model_name_or_path: Union[str, os.PathLike], ret
         </Tip>
 
         """
-        config_dict = cls.get_config_dict(pretrained_model_name_or_path=pretrained_model_name_or_path, **kwargs)
+        if not isinstance(pretrained_model_name_or_path, dict):
+            config_dict = cls.get_config_dict(pretrained_model_name_or_path=pretrained_model_name_or_path, **kwargs)
+        else:
+            config_dict = pretrained_model_name_or_path
+
         init_dict, unused_kwargs, hidden_dict = cls.extract_init_dict(config_dict, **kwargs)
 
         # Allow dtype to be specified on initialization
@@ -183,7 +179,7 @@ def from_config(cls, pretrained_model_name_or_path: Union[str, os.PathLike], ret
         model = cls(**init_dict)
 
         # make sure to also save config parameters that might be used for compatible classes
-        model.register_to_hidden_config(**hidden_dict)
+        model.register_to_config(**hidden_dict)
 
         return_tuple = (model,)
 
@@ -398,12 +394,6 @@ def __repr__(self):
     def config(self) -> Dict[str, Any]:
         return self._internal_dict
 
-    @property
-    def hidden_config(self) -> Dict[str, Any]:
-        if not hasattr(self, "_hidden_internal_dict"):
-            return {}
-        return self._hidden_internal_dict
-
     def to_json_string(self) -> str:
         """
         Serializes this instance to a JSON string.
@@ -471,6 +461,8 @@ def register_to_config(init):
     def inner_init(self, *args, **kwargs):
         # Ignore private kwargs in the init.
         init_kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_")}
+        signature = inspect.signature(init)
+
         init(self, *args, **init_kwargs)
         if not isinstance(self, ConfigMixin):
             raise RuntimeError(
@@ -481,7 +473,6 @@ def inner_init(self, *args, **kwargs):
         ignore = getattr(self, "ignore_for_config", [])
         # Get positional arguments aligned with kwargs
         new_kwargs = {}
-        signature = inspect.signature(init)
         parameters = {
             name: p.default for i, (name, p) in enumerate(signature.parameters.items()) if i > 0 and name not in ignore
         }
diff --git a/src/diffusers/pipeline_utils.py b/src/diffusers/pipeline_utils.py
index 7ea91a814fa7..4d5b07a4d1a6 100644
--- a/src/diffusers/pipeline_utils.py
+++ b/src/diffusers/pipeline_utils.py
@@ -777,13 +777,7 @@ def set_scheduler(self, scheduler_type=Union[str, SchedulerType, Dict[str, str],
                     f" {[', '.join(_compatible_class_types)]}."
                 )
 
-            # new scheduler config is current config + hidden config
-            scheduler_config = {**current_scheduler.config, **current_scheduler.hidden_config}
-
-            scheduler_init_dict, _, hidden_dict = scheduler_class.extract_init_dict(scheduler_config)
-
-            scheduler = scheduler_class(**scheduler_init_dict)
-            scheduler.register_to_hidden_config(**hidden_dict)
+            scheduler = scheduler_class.from_config(current_scheduler.config)
 
             logger.info(
                 f"Changing scheduler from type {CLASS_TO_SCHEDULER_TYPE_MAPPING[type(current_scheduler)]} to"
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
index c47ba9861499..b40605dba69a 100644
--- a/tests/test_pipelines.py
+++ b/tests/test_pipelines.py
@@ -487,9 +487,10 @@ def test_set_scheduler_consitency(self):
         )
 
         pndm_config = sd.scheduler.config
-        sd.set_scheduler("ddpm")
-        sd.set_scheduler("pndm")
+        sd.scheduler = DDPMScheduler.from_config(sd.scheduler.config)
+        sd.scheduler = PNDMScheduler.from_config(sd.scheduler.config)
         pndm_config_2 = sd.scheduler.config
+        pndm_config_2 = {k: v for k, v in pndm_config_2.items() if k in pndm_config}
 
         assert dict(pndm_config) == dict(pndm_config_2)
 
@@ -507,6 +508,7 @@ def test_set_scheduler_consitency(self):
         sd.set_scheduler("lms_discrete")
         sd.set_scheduler("ddim")
         ddim_config_2 = sd.scheduler.config
+        ddim_config_2 = {k: v for k, v in ddim_config_2.items() if k in ddim_config}
 
         assert dict(ddim_config) == dict(ddim_config_2)