[V1] Extend beyond image modality and support mixed-modality inference with Llava-OneVision

docs/source/models/supported_models.md

@@ -647,7 +647,7 @@ See [this page](#generative-models) for more information on how to use generativ
   - `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc.
   -
   - ✅︎
-  -
+  - ✅︎
 * - `MiniCPMV`
   - MiniCPM-V
   - T + I<sup>E+</sup>

tests/multimodal/test_utils.py

@@ -1,6 +1,7 @@
 import base64
 import mimetypes
 import os
+from collections import namedtuple
 from tempfile import NamedTemporaryFile, TemporaryDirectory
 from typing import Dict, Tuple
 
@@ -9,7 +10,9 @@
 from PIL import Image, ImageChops
 from transformers import AutoConfig, AutoTokenizer
 
+from vllm.multimodal.inputs import PlaceholderRange
 from vllm.multimodal.utils import (MediaConnector,
+                                   merge_and_sort_multimodal_metadata,
                                    repeat_and_pad_placeholder_tokens)
 
 # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
@@ -191,3 +194,189 @@ def test_repeat_and_pad_placeholder_tokens(model):
         assert new_prompt == expected_prompt
         assert new_token_ids == expected_token_ids
         assert ranges == expected_ranges
+
+
+# Each test case is a tuple of :
+# - mm_positions: MultiModalPlaceholderDict
+# - mm_hashes: Optional[MultiModalHashDict]
+# - expected sorted modalities: list[str]
+# - expected sorted & flattened PlaceholderRanges: list[PlaceholderRange]
+# - expected sorted & flattened hash strings: Optional[list[str]]
+TestCase = namedtuple("TestCase", [
+    "mm_positions", "mm_hashes", "expected_modalities", "expected_ranges",
+    "expected_hashes"
+])
+
+
+def test_merge_and_sort_multimodal_metadata():
+
+    test_cases = [
+        # Single modality should return result as is but flattened
+        TestCase(mm_positions={
+            "image": [
+                PlaceholderRange(offset=0, length=2),
+                PlaceholderRange(offset=3, length=2)
+            ]
+        },
+                 mm_hashes={"image": ["hash1", "hash2"]},
+                 expected_modalities=["image"],
+                 expected_ranges=[
+                     PlaceholderRange(offset=0, length=2),
+                     PlaceholderRange(offset=3, length=2)
+                 ],
+                 expected_hashes=["hash1", "hash2"]),
+        # Single modality without hashes return None for mm hash.
+        TestCase(mm_positions={
+            "image": [
+                PlaceholderRange(offset=0, length=2),
+                PlaceholderRange(offset=2, length=2)
+            ]
+        },
+                 mm_hashes=None,
+                 expected_modalities=["image"],
+                 expected_ranges=[
+                     PlaceholderRange(offset=0, length=2),
+                     PlaceholderRange(offset=2, length=2)
+                 ],
+                 expected_hashes=None),
+        # Multiple modalities with hashes should return sorted modalities
+        # and flattened ranges and hashes.
+        TestCase(mm_positions={
+            "image": [
+                PlaceholderRange(offset=7, length=4),
+                PlaceholderRange(offset=11, length=5)
+            ],
+            "audio": [
+                PlaceholderRange(offset=0, length=2),
+                PlaceholderRange(offset=2, length=3)
+            ]
+        },
+                 mm_hashes={
+                     "image": ["image_hash1", "image_hash2"],
+                     "audio": ["audio_hash1", "audio_hash2"]
+                 },
+                 expected_modalities=["audio", "image"],
+                 expected_ranges=[
+                     PlaceholderRange(offset=0, length=2),
+                     PlaceholderRange(offset=2, length=3),
+                     PlaceholderRange(offset=7, length=4),
+                     PlaceholderRange(offset=11, length=5)
+                 ],
+                 expected_hashes=[
+                     "audio_hash1", "audio_hash2", "image_hash1", "image_hash2"
+                 ]),
+        # Multiple modalities without hashes should return sorted modalities
+        # and flattened ranges and None.
+        TestCase(mm_positions={
+            "image": [
+                PlaceholderRange(offset=7, length=4),
+                PlaceholderRange(offset=11, length=5)
+            ],
+            "audio": [
+                PlaceholderRange(offset=0, length=2),
+                PlaceholderRange(offset=2, length=3)
+            ]
+        },
+                 mm_hashes=None,
+                 expected_modalities=["audio", "image"],
+                 expected_ranges=[
+                     PlaceholderRange(offset=0, length=2),
+                     PlaceholderRange(offset=2, length=3),
+                     PlaceholderRange(offset=7, length=4),
+                     PlaceholderRange(offset=11, length=5)
+                 ],
+                 expected_hashes=None),
+        # Three modalities
+        TestCase(mm_positions={
+            "image": [
+                PlaceholderRange(offset=15, length=7),
+                PlaceholderRange(offset=22, length=8),
+            ],
+            "audio": [
+                PlaceholderRange(offset=0, length=2),
+            ],
+            "video": [
+                PlaceholderRange(offset=3, length=4),
+                PlaceholderRange(offset=7, length=5),
+                PlaceholderRange(offset=12, length=6),
+            ]
+        },
+                 mm_hashes={
+                     "image": ["image_hash1", "image_hash2"],
+                     "audio": ["audio_hash1"],
+                     "video": ["video_hash1", "video_hash2", "video_hash3"]
+                 },
+                 expected_modalities=["audio", "video", "image"],
+                 expected_ranges=[
+                     PlaceholderRange(offset=0, length=2),
+                     PlaceholderRange(offset=3, length=4),
+                     PlaceholderRange(offset=7, length=5),
+                     PlaceholderRange(offset=12, length=6),
+                     PlaceholderRange(offset=15, length=7),
+                     PlaceholderRange(offset=22, length=8),
+                 ],
+                 expected_hashes=[
+                     "audio_hash1", "video_hash1", "video_hash2",
+                     "video_hash3", "image_hash1", "image_hash2"
+                 ]),
+    ]
+
+    for (mm_positions, mm_hashes, expected_modalities, expected_ranges,
+         expected_hashes) in test_cases:
+        modalities, ranges, hashes = merge_and_sort_multimodal_metadata(
+            mm_positions, mm_hashes)
+
+        assert modalities == expected_modalities
+        assert ranges == expected_ranges
+        assert hashes == expected_hashes
+
+
+def test_merge_and_sort_multimodal_metadata_with_interleaving():
+
+    test_cases = [
+
+        # <image> <audio> <image> <audio>
+        TestCase(mm_positions={
+            "image": [
+                PlaceholderRange(offset=0, length=4),
+                PlaceholderRange(offset=8, length=2)
+            ],
+            "audio": [
+                PlaceholderRange(offset=5, length=2),
+                PlaceholderRange(offset=11, length=4)
+            ]
+        },
+                 mm_hashes={
+                     "image": ["image_hash1", "image_hash2"],
+                     "audio": ["audio_hash1", "audio_hash2"]
+                 },
+                 expected_modalities=None,
+                 expected_ranges=None,
+                 expected_hashes=None),
+
+        # <image> <image> <video> <audio> <image>
+        TestCase(mm_positions={
+            "image": [
+                PlaceholderRange(offset=0, length=2),
+                PlaceholderRange(offset=2, length=3),
+                PlaceholderRange(offset=20, length=4),
+            ],
+            "audio": [
+                PlaceholderRange(offset=5, length=2),
+            ],
+            "video": [
+                PlaceholderRange(offset=8, length=5),
+            ]
+        },
+                 mm_hashes=None,
+                 expected_modalities=None,
+                 expected_ranges=None,
+                 expected_hashes=None),
+    ]
+
+    for case in test_cases:
+        with pytest.raises(ValueError) as ex_info:
+            merge_and_sort_multimodal_metadata(case.mm_positions,
+                                               case.mm_hashes)
+
+        assert "Interleaved mixed-modality" in str(ex_info.value)

vllm/model_executor/models/interfaces.py

@@ -39,8 +39,12 @@ def get_multimodal_embeddings(self, **kwargs) -> Optional[T]:
 
         The output embeddings must be one of the following formats:
         - A list or tuple of 2D tensors, where each tensor corresponds to 
-          each input image.
+          each input multimodal data item (e.g, image).
         - A single 3D tensor, with the batch dimension grouping the 2D tensors.
+
+        NOTE: The returned multimodal embeddings must be in the same order as 
+        the appearances of their corresponding multimodal data item in the 
+        input prompt.
         """
         ...
 

vllm/model_executor/models/llava_onevision.py

@@ -34,6 +34,9 @@
 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
                     maybe_prefix, merge_multimodal_embeddings)
 
+# For profile run
+_MAX_FRAMES_PER_VIDEO = 16
+
 
 class LlavaOnevisionVideoPixelInputs(TypedDict):
     type: Literal["pixel_values_videos"]
@@ -212,8 +215,10 @@ def _get_dummy_num_frames(self, seq_len: int) -> int:
         max_image_tokens = self._get_max_image_tokens() * max_images
         max_total_frames = self._get_max_video_frames(seq_len -
                                                       max_image_tokens)
+        max_frames_per_video = min(max_total_frames // max(max_videos, 1),
+                                   _MAX_FRAMES_PER_VIDEO)
 
-        return max(max_total_frames // max(max_videos, 1), 1)
+        return max(max_frames_per_video, 1)
 
     def _get_video_token(self) -> str:
         return self._get_hf_processor().video_token
@@ -520,13 +525,15 @@ def _parse_and_validate_video_input(
     def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
         modalities = {}
 
-        if "pixel_values" in kwargs:
-            modalities["images"] = self._parse_and_validate_image_input(
-                **kwargs)
-
-        if "pixel_values_videos" in kwargs:
-            modalities["videos"] = self._parse_and_validate_video_input(
-                **kwargs)
+        # Preserve the order of modalities if there are multiple of them
+        # from the order of kwargs.
+        for input_key in kwargs:
+            if input_key == "pixel_values" and "images" not in modalities:
+                modalities["images"] = self._parse_and_validate_image_input(
+                    **kwargs)
+            if input_key == "pixel_values_videos" and "videos" not in modalities:  # noqa E501
+                modalities["videos"] = self._parse_and_validate_video_input(
+                    **kwargs)
 
         return modalities
 
@@ -786,21 +793,21 @@ def get_multimodal_embeddings(
         if not modalities:
             return None
 
-        # We make a tuple of each embedding with its modality string. This is a
-        # temporary workaround for models to handle mixed modalities when
-        # get_multimodal_embeddings and get_input_embeddings are called
-        # separately.
-        # TODO(ywang96): Add support for mixed-modality inference for v1.
-        multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
-
-        if "images" in modalities:
-            image_input = modalities["images"]
-            vision_embeddings = self._process_image_input(image_input)
-            multimodal_embeddings.append((vision_embeddings, "image"))
-        if "videos" in modalities:
-            video_input = modalities["videos"]
-            video_embeddings = self._process_video_pixels(video_input)
-            multimodal_embeddings.append((video_embeddings, "video"))
+        # The result multimodal_embeddings is tuple of tensors, with each
+        # tensor correspoending to a multimodal data item (image or video).
+        multimodal_embeddings: tuple[torch.Tensor, ...] = ()
+
+        # NOTE: It is important to iterate over the keys in this dictionary
+        # to preserve the order of the modalities.
+        for modality in modalities:
+            if modality == "images":
+                image_input = modalities["images"]
+                vision_embeddings = self._process_image_input(image_input)
+                multimodal_embeddings += tuple(vision_embeddings)
+            if modality == "videos":
+                video_input = modalities["videos"]
+                video_embeddings = self._process_video_pixels(video_input)
+                multimodal_embeddings += tuple(video_embeddings)
 
         return multimodal_embeddings
 
@@ -812,15 +819,9 @@ def get_input_embeddings(
     ) -> torch.Tensor:
         inputs_embeds = self.language_model.get_input_embeddings(input_ids)
         if multimodal_embeddings is not None:
-            for embeddings, modality in multimodal_embeddings:
-                if modality == "image":
-                    inputs_embeds = merge_multimodal_embeddings(
-                        input_ids, inputs_embeds, embeddings,
-                        self.config.image_token_index)
-                if modality == "video":
-                    inputs_embeds = merge_multimodal_embeddings(
-                        input_ids, inputs_embeds, embeddings,
-                        self.config.video_token_index)
+            inputs_embeds = merge_multimodal_embeddings(
+                input_ids, inputs_embeds, multimodal_embeddings,
+                [self.config.image_token_index, self.config.video_token_index])
         return inputs_embeds
 
     def forward(

vllm/model_executor/models/molmo.py

@@ -972,8 +972,6 @@ def image_input_mapper_for_molmo(
         assert len(data) == 1, "Molmo supports only one image per prompt."
         data = data[0]
 
-    # Remove unused dummy PIL image
-    data.pop('raw_mm_data', None)
     return MultiModalKwargs(data)
 
 
@@ -1019,7 +1017,6 @@ def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
     dummy_imgdata = {
         "images": out["images"],
         "image_input_idx": out["image_input_idx"],
-        "raw_mm_data": dummy_image,
     }
     if "image_masks" in out:
         dummy_imgdata["image_masks"] = out["image_masks"]

vllm/multimodal/__init__.py

@@ -1,4 +1,5 @@
 from .base import MultiModalPlaceholderMap, MultiModalPlugin
+from .hasher import MultiModalHashDict, MultiModalHasher
 from .inputs import (BatchedTensorInputs, ModalityData, MultiModalDataBuiltins,
                      MultiModalDataDict, MultiModalKwargs,
                      MultiModalPlaceholderDict, NestedTensors)
@@ -18,6 +19,8 @@
     "ModalityData",
     "MultiModalDataBuiltins",
     "MultiModalDataDict",
+    "MultiModalHashDict",
+    "MultiModalHasher",
     "MultiModalKwargs",
     "MultiModalPlaceholderDict",
     "MultiModalPlaceholderMap",