From 6ed82359716441e8b766f24d1660b409cc323526 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 00:55:32 -0700
Subject: [PATCH 01/14] add

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 vllm/multimodal/utils.py    | 74 +++++++++++++------------------------
 vllm/v1/engine/processor.py | 51 ++++++++++---------------
 2 files changed, 46 insertions(+), 79 deletions(-)

diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py
index ad381e1d1d00..4cafb6cdb493 100644
--- a/vllm/multimodal/utils.py
+++ b/vllm/multimodal/utils.py
@@ -422,14 +422,10 @@ def merge_and_sort_multimodal_metadata(
 
     Optionally if a MultiModalHashDict is given, same operation will be 
     applied to the object and the sorted list of hashes will be returned.
-
-    Raises:
-        ValueError: If the input prompt has interleaved placeholders from
-            different modalities (e.g, "<image><audio><image> Describe the 
-            content.")
     
     Returns:
-        list[str]: Sorted list of involved modalities.
+        list[str]: List of item modalities in order of their positions in
+            the input sequence.
         list[PlaceholderRange]: Sorted list of all PlaceholdeRanges from 
             mm_positions.
         Optional[list[str]]: Sorted list of all hashes from mm_hashes if 
@@ -443,50 +439,33 @@ def merge_and_sort_multimodal_metadata(
     # For single modality, placeholder ranges and hashes are already sorted
     # so we can return the list directly.
     if len(modalities) == 1:
+        modality = modalities[0]
+        placeholder_list = list(mm_positions[modality])
         if mm_hashes is None:
-            return modalities, list(mm_positions[modalities[0]]), None
+            return [modality] * len(placeholder_list), placeholder_list, None
         else:
-            return modalities, list(mm_positions[modalities[0]]), list(
-                mm_hashes[modalities[0]])
+            return [modality] * len(placeholder_list), placeholder_list, list(
+                mm_hashes[modality])
 
-    placeholder_lists_with_modality = [(modality, mm_positions[modality])
-                                       for modality in modalities]
+    # Create a list of (modality, placeholder, hash) tuples for all placeholders
+    all_items = []
+    for modality in modalities:
+        placeholder_list = mm_positions[modality]
+        hash_list = mm_hashes[
+            modality] if mm_hashes and modality in mm_hashes else [None] * len(
+                placeholder_list)
 
-    if mm_hashes is None:
-        sorted_placeholder_lists = sorted(placeholder_lists_with_modality,
-                                          key=lambda x: x[1][0]['offset'])
-        sorted_hash_lists = None
-    else:
-        hashes_lists = [
-            mm_hashes[modality] for modality in modalities
-            if modality in mm_hashes
-        ]
-        sorted_pairs = sorted(zip(placeholder_lists_with_modality,
-                                  hashes_lists),
-                              key=lambda x: x[0][1][0]['offset'])
-        sorted_placeholder_tuple, sorted_hash_tuple = zip(*sorted_pairs)
-        sorted_placeholder_lists = list(sorted_placeholder_tuple)
-        sorted_hash_lists = list(sorted_hash_tuple)
-
-    sorted_modalities = [modality for modality, _ in sorted_placeholder_lists]
-
-    # Flatten sorted list of lists to a single list and verify there is no
-    # interleaving of placeholders from different modalities.
-    merged_placeholders: list[PlaceholderRange] = []
-    for modality, placeholder_list in sorted_placeholder_lists:
-        if merged_placeholders and placeholder_list[0][
-                'offset'] < merged_placeholders[-1]['offset']:
-            raise ValueError(
-                "Interleaved mixed-modality inference is currently not "
-                "supported.")
-        merged_placeholders.extend(placeholder_list)
-
-    if sorted_hash_lists is not None:
-        merged_hashes = []
-        for hash_list in sorted_hash_lists:
-            merged_hashes.extend(hash_list)
-    else:
-        merged_hashes = None
+        for placeholder, hash_value in zip(placeholder_list, hash_list):
+            all_items.append((modality, placeholder, hash_value))
+
+    # Sort all items by offset
+    all_items.sort(key=lambda x: x[1]['offset'])
+
+    # Split into separate lists
+    sorted_modalities = [item[0] for item in all_items]
+    merged_placeholders = [item[1] for item in all_items]
+    merged_hashes = [item[2]
+                     for item in all_items] if mm_hashes is not None else None
 
     return sorted_modalities, merged_placeholders, merged_hashes
 
@@ -502,8 +481,7 @@ def group_mm_inputs_by_modality(
 
     Returns:
         list[list[MultiModalKwargs]]: List of list of MultiModalKwargs, each 
-        inner list contains consecutive MultiModalKwargs with same modality, or
-        one with multimodal modalities.
+        inner list contains consecutive MultiModalKwargs with same modality.
     """
     if not mm_inputs:
         return []
diff --git a/vllm/v1/engine/processor.py b/vllm/v1/engine/processor.py
index ffd12d5fd0d8..26622719d6c8 100644
--- a/vllm/v1/engine/processor.py
+++ b/vllm/v1/engine/processor.py
@@ -238,22 +238,11 @@ def process_inputs(
         if (decoder_mm_inputs := decoder_inputs.multi_modal_data):
             assert isinstance(decoder_mm_inputs, MultiModalKwargs)
 
-            # The output of merged multi-modal processor (`decoder_mm_inputs`)
-            # contains the kwargs for all items from all modalities.
-            # This code separates them so that there is one set of kwargs
-            # per item per modality.
-            individual_mm_inputs = [
-                MultiModalKwargs.from_items([item])
-                for modality in decoder_mm_inputs.modalities
-                for item in decoder_mm_inputs.get_items(modality)
-            ]
-
             # Merge and flatten multimodal placeholders, hashes and inputs
             # from dictionaries to lists, and sort them by each item's position
             # in the input sequence.
-            # NOTE: interleaved modalities are not supported.
             (
-                sorted_modalities,
+                sorted_item_modalities,
                 sorted_mm_positions,
                 sorted_mm_hashes,
             ) = merge_and_sort_multimodal_metadata(
@@ -261,26 +250,26 @@ def process_inputs(
                 decoder_inputs.multi_modal_hashes if self.use_hash else None,
             )
 
-            # NOTE: Sort multimodal inputs/kwargs ONLY IF there are multiple
-            # modalities involved.
-            if len(sorted_modalities) > 1:
-                modality_order_dict = {
-                    modality: order
-                    for order, modality in enumerate(sorted_modalities)
-                }
-
-                # Sanity check to make sure each multimodal input has only one
-                # modality key.
-                for mm_input in individual_mm_inputs:
-                    assert len(mm_input.modalities) == 1
-
-                # Sort MultiModalKwargs to match sorted_mm_positions
-                sorted_mm_inputs = sorted(
-                    individual_mm_inputs,
-                    key=lambda mm_input: modality_order_dict[list(
-                        mm_input.modalities)[0]])
+            # The output of merged multi-modal processor (`decoder_mm_inputs`)
+            # is a single MultiModalKwargs for all items from all modalities.
+            # This code flattens kwargs for individual items in a list and
+            # sorts them by each item's position in the input sequence if there
+            # are multiple modalities.
+            unique_modalities = set(sorted_item_modalities)
+            if len(unique_modalities) > 1:
+                sorted_mm_inputs = []
+                used_indices = {modality: 0 for modality in unique_modalities}
+                for modality in sorted_item_modalities:
+                    items = decoder_mm_inputs.get_items(modality)
+                    item = items[used_indices[modality]]
+                    sorted_mm_inputs.append(MultiModalKwargs.from_items([item
+                                                                         ]))
+                    used_indices[modality] += 1
             else:
-                sorted_mm_inputs = individual_mm_inputs
+                sorted_mm_inputs = [
+                    MultiModalKwargs.from_items([item]) for item in
+                    decoder_mm_inputs.get_items(sorted_item_modalities[0])
+                ]
 
         return EngineCoreRequest(
             request_id=request_id,

From 2b0cffa74a1644ad34fd40af0b365a6dbdeb5fc8 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 01:00:31 -0700
Subject: [PATCH 02/14] update test

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/multimodal/test_utils.py | 52 +++++++++++++++++++++++-----------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/tests/multimodal/test_utils.py b/tests/multimodal/test_utils.py
index 8f76d895fdd2..911b5f2dafff 100644
--- a/tests/multimodal/test_utils.py
+++ b/tests/multimodal/test_utils.py
@@ -222,7 +222,7 @@ def test_merge_and_sort_multimodal_metadata():
                 ]
             },
             mm_hashes={"image": ["hash1", "hash2"]},
-            expected_modalities=["image"],
+            expected_modalities=["image", "image"],
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
                 PlaceholderRange(offset=3, length=2),
@@ -239,7 +239,7 @@ def test_merge_and_sort_multimodal_metadata():
                 ]
             },
             mm_hashes=None,
-            expected_modalities=["image"],
+            expected_modalities=["image", "image"],
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
                 PlaceholderRange(offset=2, length=2),
@@ -264,7 +264,7 @@ def test_merge_and_sort_multimodal_metadata():
                 "image": ["image_hash1", "image_hash2"],
                 "audio": ["audio_hash1", "audio_hash2"],
             },
-            expected_modalities=["audio", "image"],
+            expected_modalities=["audio", "audio", "image", "image"],
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
                 PlaceholderRange(offset=2, length=3),
@@ -290,7 +290,7 @@ def test_merge_and_sort_multimodal_metadata():
                 ]
             },
             mm_hashes=None,
-            expected_modalities=["audio", "image"],
+            expected_modalities=["audio", "audio", "image", "image"],
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
                 PlaceholderRange(offset=2, length=3),
@@ -321,7 +321,9 @@ def test_merge_and_sort_multimodal_metadata():
                 "audio": ["audio_hash1"],
                 "video": ["video_hash1", "video_hash2", "video_hash3"]
             },
-            expected_modalities=["audio", "video", "image"],
+            expected_modalities=[
+                "audio", "audio", "video", "video", "video", "image", "image"
+            ],
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
                 PlaceholderRange(offset=3, length=4),
@@ -367,9 +369,16 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
                 "image": ["image_hash1", "image_hash2"],
                 "audio": ["audio_hash1", "audio_hash2"],
             },
-            expected_modalities=[],
-            expected_ranges=[],
-            expected_hashes=None,
+            expected_modalities=["image", "audio", "image", "audio"],
+            expected_ranges=[
+                PlaceholderRange(offset=0, length=4),
+                PlaceholderRange(offset=5, length=2),
+                PlaceholderRange(offset=8, length=2),
+                PlaceholderRange(offset=11, length=4),
+            ],
+            expected_hashes=[
+                "image_hash1", "audio_hash1", "image_hash2", "audio_hash2"
+            ],
         ),
 
         # <image> <image> <video> <audio> <image>
@@ -388,15 +397,26 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
                 ]
             },
             mm_hashes=None,
-            expected_modalities=[],
-            expected_ranges=[],
-            expected_hashes=None,
+            expected_modalities=["image", "image", "video", "audio", "image"],
+            expected_ranges=[
+                PlaceholderRange(offset=0, length=2),
+                PlaceholderRange(offset=2, length=3),
+                PlaceholderRange(offset=8, length=5),
+                PlaceholderRange(offset=5, length=2),
+                PlaceholderRange(offset=20, length=4),
+            ],
+            expected_hashes=[
+                "image_hash1", "image_hash2", "video_hash1", "audio_hash1",
+                "image_hash3"
+            ],
         ),
     ]
 
-    for case in test_cases:
-        with pytest.raises(ValueError) as ex_info:
-            merge_and_sort_multimodal_metadata(case.mm_positions,
-                                               case.mm_hashes)
+    for (mm_positions, mm_hashes, expected_modalities, expected_ranges,
+         expected_hashes) in test_cases:
+        modalities, ranges, hashes = merge_and_sort_multimodal_metadata(
+            mm_positions, mm_hashes)
 
-        assert "Interleaved mixed-modality" in str(ex_info.value)
+        assert modalities == expected_modalities
+        assert ranges == expected_ranges
+        assert hashes == expected_hashes

From 7d82cba97402fa8523202cbe0c30a0903e77bbe8 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 01:04:16 -0700
Subject: [PATCH 03/14] fix test

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/multimodal/test_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/multimodal/test_utils.py b/tests/multimodal/test_utils.py
index 911b5f2dafff..58e67cca8ca4 100644
--- a/tests/multimodal/test_utils.py
+++ b/tests/multimodal/test_utils.py
@@ -381,7 +381,7 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
             ],
         ),
 
-        # <image> <image> <video> <audio> <image>
+        # <image> <image> <audio> <video> <image>
         TestCase(
             mm_positions={
                 "image": [
@@ -397,7 +397,7 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
                 ]
             },
             mm_hashes=None,
-            expected_modalities=["image", "image", "video", "audio", "image"],
+            expected_modalities=["image", "image", "audio", "video", "image"],
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
                 PlaceholderRange(offset=2, length=3),

From 68bf3913ec42bd64b7215c3047e853271db200d4 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 01:05:19 -0700
Subject: [PATCH 04/14] update test

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/multimodal/test_utils.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/multimodal/test_utils.py b/tests/multimodal/test_utils.py
index 58e67cca8ca4..43af10a0cda3 100644
--- a/tests/multimodal/test_utils.py
+++ b/tests/multimodal/test_utils.py
@@ -405,10 +405,7 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
                 PlaceholderRange(offset=5, length=2),
                 PlaceholderRange(offset=20, length=4),
             ],
-            expected_hashes=[
-                "image_hash1", "image_hash2", "video_hash1", "audio_hash1",
-                "image_hash3"
-            ],
+            expected_hashes=None,
         ),
     ]
 

From 5d3b9a695db10b5577fedfa0f5077f3f7c3ab55b Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 01:07:58 -0700
Subject: [PATCH 05/14] fix test

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/multimodal/test_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/multimodal/test_utils.py b/tests/multimodal/test_utils.py
index 43af10a0cda3..0c126ba17de4 100644
--- a/tests/multimodal/test_utils.py
+++ b/tests/multimodal/test_utils.py
@@ -322,7 +322,7 @@ def test_merge_and_sort_multimodal_metadata():
                 "video": ["video_hash1", "video_hash2", "video_hash3"]
             },
             expected_modalities=[
-                "audio", "audio", "video", "video", "video", "image", "image"
+                "audio", "video", "video", "video", "image", "image"
             ],
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
@@ -401,8 +401,8 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
             expected_ranges=[
                 PlaceholderRange(offset=0, length=2),
                 PlaceholderRange(offset=2, length=3),
-                PlaceholderRange(offset=8, length=5),
                 PlaceholderRange(offset=5, length=2),
+                PlaceholderRange(offset=8, length=5),
                 PlaceholderRange(offset=20, length=4),
             ],
             expected_hashes=None,

From 240bdb4d16e1569588ed406d955a757b6cb4b647 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 01:38:58 -0700
Subject: [PATCH 06/14] mypy

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 vllm/multimodal/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py
index 4cafb6cdb493..c43a3c7ab3de 100644
--- a/vllm/multimodal/utils.py
+++ b/vllm/multimodal/utils.py
@@ -440,7 +440,7 @@ def merge_and_sort_multimodal_metadata(
     # so we can return the list directly.
     if len(modalities) == 1:
         modality = modalities[0]
-        placeholder_list = list(mm_positions[modality])
+        placeholder_list = mm_positions[modality]
         if mm_hashes is None:
             return [modality] * len(placeholder_list), placeholder_list, None
         else:
@@ -451,7 +451,7 @@ def merge_and_sort_multimodal_metadata(
     all_items = []
     for modality in modalities:
         placeholder_list = mm_positions[modality]
-        hash_list = mm_hashes[
+        hash_list: list[Optional[str]] = mm_hashes[
             modality] if mm_hashes and modality in mm_hashes else [None] * len(
                 placeholder_list)
 

From 5bb5646e2b93a83b486dafdb1aeae07e75bb0b91 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 01:54:36 -0700
Subject: [PATCH 07/14] mypy

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 vllm/multimodal/utils.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py
index c43a3c7ab3de..e8b2217ab46e 100644
--- a/vllm/multimodal/utils.py
+++ b/vllm/multimodal/utils.py
@@ -440,7 +440,7 @@ def merge_and_sort_multimodal_metadata(
     # so we can return the list directly.
     if len(modalities) == 1:
         modality = modalities[0]
-        placeholder_list = mm_positions[modality]
+        placeholder_list = list(mm_positions[modality])
         if mm_hashes is None:
             return [modality] * len(placeholder_list), placeholder_list, None
         else:
@@ -450,10 +450,11 @@ def merge_and_sort_multimodal_metadata(
     # Create a list of (modality, placeholder, hash) tuples for all placeholders
     all_items = []
     for modality in modalities:
-        placeholder_list = mm_positions[modality]
-        hash_list: list[Optional[str]] = mm_hashes[
-            modality] if mm_hashes and modality in mm_hashes else [None] * len(
-                placeholder_list)
+        placeholder_list = list(mm_positions[modality])
+        hash_list: list[Optional[str]] = list(
+            mm_hashes[modality]) if mm_hashes and modality in mm_hashes else [
+                None
+            ] * len(placeholder_list)
 
         for placeholder, hash_value in zip(placeholder_list, hash_list):
             all_items.append((modality, placeholder, hash_value))
@@ -464,7 +465,7 @@ def merge_and_sort_multimodal_metadata(
     # Split into separate lists
     sorted_modalities = [item[0] for item in all_items]
     merged_placeholders = [item[1] for item in all_items]
-    merged_hashes = [item[2]
+    merged_hashes = [str(item[2])
                      for item in all_items] if mm_hashes is not None else None
 
     return sorted_modalities, merged_placeholders, merged_hashes

From 429a06c646f59597f436f5c9c7ce445cc3340d1f Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 02:30:39 -0700
Subject: [PATCH 08/14] simplify

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 vllm/multimodal/utils.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py
index e8b2217ab46e..1b76b0950f84 100644
--- a/vllm/multimodal/utils.py
+++ b/vllm/multimodal/utils.py
@@ -441,11 +441,10 @@ def merge_and_sort_multimodal_metadata(
     if len(modalities) == 1:
         modality = modalities[0]
         placeholder_list = list(mm_positions[modality])
-        if mm_hashes is None:
-            return [modality] * len(placeholder_list), placeholder_list, None
-        else:
-            return [modality] * len(placeholder_list), placeholder_list, list(
-                mm_hashes[modality])
+
+        return [modality] * len(
+            placeholder_list
+        ), placeholder_list, None if not mm_hashes else mm_hashes[modality]
 
     # Create a list of (modality, placeholder, hash) tuples for all placeholders
     all_items = []

From 07325c92d60e8d8cca8e286b0fa3e89856e5595b Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Thu, 27 Mar 2025 02:35:36 -0700
Subject: [PATCH 09/14] add test

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/multimodal/test_utils.py | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/tests/multimodal/test_utils.py b/tests/multimodal/test_utils.py
index 0c126ba17de4..b50e7d400b07 100644
--- a/tests/multimodal/test_utils.py
+++ b/tests/multimodal/test_utils.py
@@ -407,6 +407,37 @@ def test_merge_and_sort_multimodal_metadata_with_interleaving():
             ],
             expected_hashes=None,
         ),
+
+        # <image> <audio> <video> <image> with hashes
+        TestCase(
+            mm_positions={
+                "image": [
+                    PlaceholderRange(offset=0, length=2),
+                    PlaceholderRange(offset=18, length=4),
+                ],
+                "audio": [
+                    PlaceholderRange(offset=6, length=2),
+                ],
+                "video": [
+                    PlaceholderRange(offset=10, length=5),
+                ]
+            },
+            mm_hashes={
+                "image": ["image_hash1", "image_hash2"],
+                "audio": ["audio_hash1"],
+                "video": ["video_hash1"],
+            },
+            expected_modalities=["image", "audio", "video", "image"],
+            expected_ranges=[
+                PlaceholderRange(offset=0, length=2),
+                PlaceholderRange(offset=6, length=2),
+                PlaceholderRange(offset=10, length=5),
+                PlaceholderRange(offset=18, length=4),
+            ],
+            expected_hashes=[
+                "image_hash1", "audio_hash1", "video_hash1", "image_hash2"
+            ],
+        ),
     ]
 
     for (mm_positions, mm_hashes, expected_modalities, expected_ranges,

From 2f1228e0b62069c57f22ca1c7a88245e2cef8cfd Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Fri, 28 Mar 2025 17:21:26 -0700
Subject: [PATCH 10/14] add mixed-modality test runner

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/conftest.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index cc48fceb8eff..453be39088eb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -757,20 +757,17 @@ def get_inputs(
             assert len(prompts) == len(audios)
 
         inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
-        if images is not None:
-            for i, image in enumerate(images):
-                if image is not None:
-                    inputs[i]["multi_modal_data"] = {"image": image}
 
-        if videos is not None:
-            for i, video in enumerate(videos):
-                if video is not None:
-                    inputs[i]["multi_modal_data"] = {"video": video}
+        for i in range(len(inputs)):
+            inputs[i]["multi_modal_data"] = {}
+            if images is not None and (image := images[i]) is not None:
+                inputs[i]["multi_modal_data"]["image"] = image
 
-        if audios is not None:
-            for i, audio in enumerate(audios):
-                if audio is not None:
-                    inputs[i]["multi_modal_data"] = {"audio": audio}
+            if videos is not None and (video := videos[i]) is not None:
+                inputs[i]["multi_modal_data"]["video"] = video
+
+            if audios is not None and (audio := audios[i]) is not None:
+                inputs[i]["multi_modal_data"]["audio"] = audio
 
         return inputs
 

From 92ab8dd35bd764c968f00b83a87378c7fad55691 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Sat, 29 Mar 2025 00:20:27 -0700
Subject: [PATCH 11/14] add simple test

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 .../vision_language/test_interleaved.py       | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 tests/models/decoder_only/vision_language/test_interleaved.py

diff --git a/tests/models/decoder_only/vision_language/test_interleaved.py b/tests/models/decoder_only/vision_language/test_interleaved.py
new file mode 100644
index 000000000000..99b6a72bdc6f
--- /dev/null
+++ b/tests/models/decoder_only/vision_language/test_interleaved.py
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from vllm.assets.image import ImageAsset
+from vllm.assets.video import VideoAsset
+
+models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
+
+
+def base_prompt(modalities_str: str) -> str:
+    return f"<|im_start|>user {modalities_str}\nDescribe what you see from these items.<|im_end|><|im_start|>assistant\n"  # noqa: E501
+
+
+INTERLEAVED_PROMPT = base_prompt("<image><video><image>\n")
+NONINTERLEAVED_PROMPT = base_prompt("<image><image><video>\n")
+
+
+@pytest.mark.parametrize("model", models)
+@pytest.mark.parametrize("dtype", ["float16"])
+@pytest.mark.parametrize("max_tokens", [128])
+def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
+    """
+    This is a simple test to check if interleaved and non-interleaved prompts
+    give the same result.
+    """
+
+    image_cherry = ImageAsset("cherry_blossom").pil_image.convert("RGB")
+    image_stop = ImageAsset("stop_sign").pil_image.convert("RGB")
+    images = [image_cherry, image_stop]
+    video = VideoAsset(name="sample_demo_1.mp4", num_frames=16).np_ndarrays
+
+    inputs = [
+        (
+            [INTERLEAVED_PROMPT],
+            [images],
+            [video],
+        ),
+        (
+            [NONINTERLEAVED_PROMPT],
+            [images],
+            [video],
+        ),
+    ]
+
+    with vllm_runner(model,
+                     task="generate",
+                     dtype=dtype,
+                     limit_mm_per_prompt={"image": 2},
+                     max_model_len=32768,
+                     max_num_seqs=2,
+                     tensor_parallel_size=1,
+                     enforce_eager=True) as vllm_model:
+        vllm_outputs_per_case = [
+            vllm_model.generate_greedy(prompts,
+                                       max_tokens,
+                                       images=images,
+                                       videos=videos)
+            for prompts, images, videos in inputs
+        ]
+
+    all_results = [output[0][1] for output in vllm_outputs_per_case]
+    outputs = [(total_str, total_str.find("assistant\n") + len("assistant\n"))
+               for total_str in all_results]
+    prompt_lengths = [prompt_len for _, prompt_len in outputs]
+    generated_strs = [
+        total_str[prompt_len:] for total_str, prompt_len in outputs
+    ]
+    interleaved_prompt_len, noninterleaved_prompt_len = prompt_lengths
+    interleaved_output_str, noninterleaved_output_str = generated_strs
+
+    # The two prompts are identical except for the modality tokens.
+    assert interleaved_prompt_len == noninterleaved_prompt_len
+
+    # The two generated strings should be different because of the
+    # interleaved modality tokens.
+    assert interleaved_output_str != noninterleaved_output_str

From d18eb2de09892117772a06df5c5fe39fa1dbb625 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Sat, 29 Mar 2025 00:22:17 -0700
Subject: [PATCH 12/14] add to standard test

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 .buildkite/test-pipeline.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml
index f22b2b0ab6f2..b8c64cf578f7 100644
--- a/.buildkite/test-pipeline.yaml
+++ b/.buildkite/test-pipeline.yaml
@@ -429,6 +429,7 @@ steps:
     - pytest -v -s models/encoder_decoder/audio_language -m core_model
     - pytest -v -s models/encoder_decoder/language -m core_model
     - pytest -v -s models/encoder_decoder/vision_language -m core_model
+    - pytest -v -s models/decoder_only/vision_language/test_interleaved.py
 
 - label: Multi-Modal Models Test (Extended) 1 # 48m
   optional: true

From f8f0efd92a83384dbdb7518c69abd63a4e7e5fd1 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Sat, 29 Mar 2025 00:26:09 -0700
Subject: [PATCH 13/14] comment

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/models/decoder_only/vision_language/test_interleaved.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/models/decoder_only/vision_language/test_interleaved.py b/tests/models/decoder_only/vision_language/test_interleaved.py
index 99b6a72bdc6f..8804497ae616 100644
--- a/tests/models/decoder_only/vision_language/test_interleaved.py
+++ b/tests/models/decoder_only/vision_language/test_interleaved.py
@@ -69,7 +69,7 @@ def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:
     interleaved_prompt_len, noninterleaved_prompt_len = prompt_lengths
     interleaved_output_str, noninterleaved_output_str = generated_strs
 
-    # The two prompts are identical except for the modality tokens.
+    # The two prompts are identical except for the order of modality tokens.
     assert interleaved_prompt_len == noninterleaved_prompt_len
 
     # The two generated strings should be different because of the

From 98e4163bc30a2c93d00bd1acb625f7c4f0ed5b60 Mon Sep 17 00:00:00 2001
From: Roger Wang <ywang@roblox.com>
Date: Sat, 29 Mar 2025 02:10:46 -0700
Subject: [PATCH 14/14] simplify

Signed-off-by: Roger Wang <ywang@roblox.com>
---
 tests/conftest.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 453be39088eb..6627ab638bf5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -747,27 +747,27 @@ def get_inputs(
         videos: Optional[PromptVideoInput] = None,
         audios: Optional[PromptAudioInput] = None,
     ) -> list[TextPrompt]:
-        if images is not None:
-            assert len(prompts) == len(images)
-
-        if videos is not None:
-            assert len(prompts) == len(videos)
-
-        if audios is not None:
-            assert len(prompts) == len(audios)
 
-        inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
+        if any(x is not None and len(x) != len(prompts)
+               for x in [images, videos, audios]):
+            raise ValueError(
+                "All non-None multimodal inputs must have the same length as "
+                "prompts")
 
-        for i in range(len(inputs)):
-            inputs[i]["multi_modal_data"] = {}
+        inputs = []
+        for i, prompt in enumerate(prompts):
+            multi_modal_data = {}
             if images is not None and (image := images[i]) is not None:
-                inputs[i]["multi_modal_data"]["image"] = image
-
+                multi_modal_data["image"] = image
             if videos is not None and (video := videos[i]) is not None:
-                inputs[i]["multi_modal_data"]["video"] = video
-
+                multi_modal_data["video"] = video
             if audios is not None and (audio := audios[i]) is not None:
-                inputs[i]["multi_modal_data"]["audio"] = audio
+                multi_modal_data["audio"] = audio
+
+            inputs.append(
+                TextPrompt(prompt=prompt,
+                           multi_modal_data=multi_modal_data
+                           if multi_modal_data else None))
 
         return inputs