diff --git a/data/dataset_info.json b/data/dataset_info.json
index b00456d2a7..8a1b4356e4 100644
--- a/data/dataset_info.json
+++ b/data/dataset_info.json
@@ -38,6 +38,35 @@
       "assistant_tag": "assistant"
     }
   },
+  "video_demo": {
+    "file_name": "video_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "videos": "videos"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
+  "visual_mix_demo": {
+    "file_name": "visual_mix_demo.json",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages",
+      "videos": "videos",
+      "images": "images"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant"
+    }
+  },
   "alpaca_en": {
     "hf_hub_url": "llamafactory/alpaca_en",
     "ms_hub_url": "llamafactory/alpaca_en"
diff --git a/data/mllm_demo_data/1.mp4 b/data/mllm_demo_data/1.mp4
new file mode 100644
index 0000000000..8afbc6c561
Binary files /dev/null and b/data/mllm_demo_data/1.mp4 differ
diff --git a/data/mllm_demo_data/2.avi b/data/mllm_demo_data/2.avi
new file mode 100644
index 0000000000..bdb736c256
Binary files /dev/null and b/data/mllm_demo_data/2.avi differ
diff --git a/data/mllm_demo_data/3.mp4 b/data/mllm_demo_data/3.mp4
new file mode 100644
index 0000000000..17cc120e9f
Binary files /dev/null and b/data/mllm_demo_data/3.mp4 differ
diff --git a/data/video_demo.json b/data/video_demo.json
new file mode 100644
index 0000000000..1912edb88b
--- /dev/null
+++ b/data/video_demo.json
@@ -0,0 +1,47 @@
+[
+  {
+    "messages": [
+      {
+        "content": "Why is this video funny?",
+        "role": "user"
+      },
+      {
+        "content": "Because a baby is reading, and he is so cute!",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/1.mp4"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "What is she doing?",
+        "role": "user"
+      },
+      {
+        "content": "She is cooking",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/2.avi"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "What's in the video?",
+        "role": "user"
+      },
+      {
+        "content": "A baby is playing in the living room",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/3.mp4"
+    ]
+  }
+]
\ No newline at end of file
diff --git a/data/visual_mix_demo.json b/data/visual_mix_demo.json
new file mode 100644
index 0000000000..cfb6d31457
--- /dev/null
+++ b/data/visual_mix_demo.json
@@ -0,0 +1,56 @@
+[
+  {
+    "messages": [
+      {
+        "content": "<video>Why is this video funny?<image>",
+        "role": "user"
+      },
+      {
+        "content": "Because a baby is reading, and he is so cute!",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/1.mp4"
+    ],
+    "images": [
+      "mllm_demo_data/1.jpg"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<video>What is she doing?<image>",
+        "role": "user"
+      },
+      {
+        "content": "She is cooking",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/2.avi"
+    ],
+    "images": [
+      "mllm_demo_data/2.jpg"
+    ]
+  },
+  {
+    "messages": [
+      {
+        "content": "<video>Why is this video funny?<image>",
+        "role": "user"
+      },
+      {
+        "content": "A baby is playing!",
+        "role": "assistant"
+      }
+    ],
+    "videos": [
+      "mllm_demo_data/3.mp4"
+    ],
+    "images": [
+      "mllm_demo_data/3.jpg"
+    ]
+  }
+]
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 62e77e1f4e..60719c55a3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -22,7 +22,8 @@ known-third-party = [
     "peft",
     "torch",
     "transformers",
-    "trl"
+    "trl",
+    "av",
 ]
 
 [tool.ruff.format]
diff --git a/requirements.txt b/requirements.txt
index 93e83530c0..e9afe326e6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,4 +18,5 @@ matplotlib>=3.7.0
 fire
 packaging
 pyyaml
+av
 numpy<2.0.0
diff --git a/src/llamafactory/chat/base_engine.py b/src/llamafactory/chat/base_engine.py
index ccdf4c92a5..5e7a586ea5 100644
--- a/src/llamafactory/chat/base_engine.py
+++ b/src/llamafactory/chat/base_engine.py
@@ -57,6 +57,7 @@ async def chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         **input_kwargs,
     ) -> List["Response"]: ...
 
@@ -67,6 +68,7 @@ async def stream_chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         **input_kwargs,
     ) -> AsyncGenerator[str, None]: ...
 
diff --git a/src/llamafactory/chat/chat_model.py b/src/llamafactory/chat/chat_model.py
index 3ea3b44fb3..125be83667 100644
--- a/src/llamafactory/chat/chat_model.py
+++ b/src/llamafactory/chat/chat_model.py
@@ -78,9 +78,10 @@ def stream_chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         **input_kwargs,
     ) -> Generator[str, None, None]:
-        generator = self.astream_chat(messages, system, tools, image, **input_kwargs)
+        generator = self.astream_chat(messages, system, tools, image, video, **input_kwargs)
         while True:
             try:
                 task = asyncio.run_coroutine_threadsafe(generator.__anext__(), self._loop)
@@ -94,9 +95,10 @@ async def astream_chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         **input_kwargs,
     ) -> AsyncGenerator[str, None]:
-        async for new_token in self.engine.stream_chat(messages, system, tools, image, **input_kwargs):
+        async for new_token in self.engine.stream_chat(messages, system, tools, image, video, **input_kwargs):
             yield new_token
 
     def get_scores(
diff --git a/src/llamafactory/chat/hf_engine.py b/src/llamafactory/chat/hf_engine.py
index 6e728c2b6a..151484bfb4 100644
--- a/src/llamafactory/chat/hf_engine.py
+++ b/src/llamafactory/chat/hf_engine.py
@@ -26,12 +26,11 @@
 from ..extras.misc import get_logits_processor
 from ..model import load_model, load_tokenizer
 from .base_engine import BaseEngine, Response
-
+from ..data.processors.processor_utils import get_pixel_values, get_pixel_values_videos
 
 if TYPE_CHECKING:
     from numpy.typing import NDArray
     from transformers import PreTrainedModel, PreTrainedTokenizer, ProcessorMixin
-    from transformers.image_processing_utils import BaseImageProcessor
     from trl import PreTrainedModelWrapper
 
     from ..data import Template
@@ -79,30 +78,58 @@ def _process_args(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         input_kwargs: Optional[Dict[str, Any]] = {},
     ) -> Tuple[Dict[str, Any], int]:
+        processor_class = "" if processor is None else type(processor).__name__
+
+        visual_token_flag = template.image_token in messages[0]["content"] or \
+                            template.video_token in messages[0]["content"]
+
         if (
             processor is not None
             and image is not None
-            and not hasattr(processor, "image_seq_length")
+            and not processor_class == 'PaliGemmaProcessor'
             and template.image_token not in messages[0]["content"]
-        ):  # llava-like models
+            and not visual_token_flag
+        ):
             messages[0]["content"] = template.image_token + messages[0]["content"]
 
+        if (
+            processor is not None
+            and video is not None
+            and template.video_token not in messages[0]["content"]
+            and not visual_token_flag
+        ):
+            messages[0]["content"] = template.video_token + messages[0]["content"]
+
+        if processor_class == 'Idefics2Processor':
+            fake_image_token = processor.fake_image_token.content
+            image_str = f"{fake_image_token}{template.image_token * processor.image_seq_len}{fake_image_token}"
+            image_str = image_str * 5
+            for j in range(len(messages)):
+                content = messages[j]['content']
+                content = content.replace(template.image_token, image_str)
+                content = content.replace(f"{fake_image_token}{fake_image_token}", f"{fake_image_token}")
+                messages[j]['content'] = content
+
         paired_messages = messages + [{"role": "assistant", "content": ""}]
         system = system or generating_args["default_system"]
         pixel_values = None
+        pixel_values_video = None
         prompt_ids, _ = template.encode_oneturn(
             tokenizer=tokenizer, messages=paired_messages, system=system, tools=tools
         )
+
         if processor is not None and image is not None:  # add image features
-            image_processor: "BaseImageProcessor" = getattr(processor, "image_processor")
-            batch_feature = image_processor(image, return_tensors="pt")
-            pixel_values = batch_feature.to(model.device)["pixel_values"]  # shape (B, C, H, W)
-            if hasattr(processor, "image_seq_length"):  # paligemma models
+            pixel_values = get_pixel_values([image], processor, template.image_data_key)
+            if processor_class == 'PaliGemmaProcessor':  # paligemma models
                 image_token_id = tokenizer.convert_tokens_to_ids(template.image_token)
                 prompt_ids = [image_token_id] * getattr(processor, "image_seq_length") + prompt_ids
 
+        if processor is not None and video is not None:  # add video features
+            pixel_values_video = get_pixel_values_videos([video], processor, template.video_data_key)
+
         prompt_length = len(prompt_ids)
         inputs = torch.tensor([prompt_ids], device=model.device)
         attention_mask = torch.ones_like(inputs, dtype=torch.bool)
@@ -165,7 +192,12 @@ def _process_args(
         )
 
         if pixel_values is not None:
-            gen_kwargs["pixel_values"] = pixel_values
+            for key in template.image_data_key:
+                gen_kwargs[key] = pixel_values[key].to(model.device)
+
+        if pixel_values_video is not None:
+            for key in template.video_data_key:
+                gen_kwargs[key] = pixel_values_video[key].to(model.device)
 
         return gen_kwargs, prompt_length
 
@@ -181,10 +213,11 @@ def _chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         input_kwargs: Optional[Dict[str, Any]] = {},
     ) -> List["Response"]:
         gen_kwargs, prompt_length = HuggingfaceEngine._process_args(
-            model, tokenizer, processor, template, generating_args, messages, system, tools, image, input_kwargs
+            model, tokenizer, processor, template, generating_args, messages, system, tools, image, video, input_kwargs
         )
         generate_output = model.generate(**gen_kwargs)
         response_ids = generate_output[:, prompt_length:]
@@ -216,10 +249,11 @@ def _stream_chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         input_kwargs: Optional[Dict[str, Any]] = {},
     ) -> Callable[[], str]:
         gen_kwargs, _ = HuggingfaceEngine._process_args(
-            model, tokenizer, processor, template, generating_args, messages, system, tools, image, input_kwargs
+            model, tokenizer, processor, template, generating_args, messages, system, tools, image, video, input_kwargs
         )
         streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
         gen_kwargs["streamer"] = streamer
@@ -273,6 +307,7 @@ async def chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         **input_kwargs,
     ) -> List["Response"]:
         if not self.can_generate:
@@ -289,6 +324,7 @@ async def chat(
             system,
             tools,
             image,
+            video,
             input_kwargs,
         )
         async with self.semaphore:
@@ -301,6 +337,7 @@ async def stream_chat(
         system: Optional[str] = None,
         tools: Optional[str] = None,
         image: Optional["NDArray"] = None,
+        video: Optional["NDArray"] = None,
         **input_kwargs,
     ) -> AsyncGenerator[str, None]:
         if not self.can_generate:
@@ -317,6 +354,7 @@ async def stream_chat(
             system,
             tools,
             image,
+            video,
             input_kwargs,
         )
         async with self.semaphore:
diff --git a/src/llamafactory/data/aligner.py b/src/llamafactory/data/aligner.py
index ef70d75b72..d739021327 100644
--- a/src/llamafactory/data/aligner.py
+++ b/src/llamafactory/data/aligner.py
@@ -48,14 +48,30 @@ def _convert_images(images: List[Any], dataset_attr: "DatasetAttr", data_args: "
     return outputs
 
 
+def _convert_videos(videos: List[Any], dataset_attr: "DatasetAttr", data_args: "DataArguments") -> List[Any]:
+    r"""
+    Optionally concatenates image path to dataset dir when loading from local disk.
+    """
+    outputs = []
+    if dataset_attr.load_from in ["script", "file"]:
+        for video in videos:
+            if isinstance(video, str) and os.path.isfile(os.path.join(data_args.dataset_dir, video)):
+                outputs.append(os.path.join(data_args.dataset_dir, video))
+            else:
+                outputs.append(video)
+
+    return outputs
+
+
 def convert_alpaca(
     examples: Dict[str, List[Any]], dataset_attr: "DatasetAttr", data_args: "DataArguments"
 ) -> Dict[str, List[Any]]:
     r"""
     Converts alpaca format dataset to the standard format.
     """
-    outputs = {"prompt": [], "response": [], "system": [], "tools": [], "images": []}
+    outputs = {"prompt": [], "response": [], "system": [], "tools": [], "images": [], "videos": []}
     convert_images = partial(_convert_images, dataset_attr=dataset_attr, data_args=data_args)
+    convert_videos = partial(_convert_videos, dataset_attr=dataset_attr, data_args=data_args)
     for i in range(len(examples[dataset_attr.prompt])):
         prompt = []
         if dataset_attr.history and isinstance(examples[dataset_attr.history][i], list):
@@ -97,6 +113,7 @@ def convert_alpaca(
         outputs["system"].append(examples[dataset_attr.system][i] if dataset_attr.system else "")
         outputs["tools"].append(examples[dataset_attr.tools][i] if dataset_attr.tools else "")
         outputs["images"].append(convert_images(examples[dataset_attr.images][i]) if dataset_attr.images else [])
+        outputs["videos"].append(convert_videos(examples[dataset_attr.videos][i]) if dataset_attr.videos else [])
 
     return outputs
 
@@ -107,8 +124,9 @@ def convert_sharegpt(
     r"""
     Converts sharegpt format dataset to the standard format.
     """
-    outputs = {"prompt": [], "response": [], "system": [], "tools": [], "images": []}
+    outputs = {"prompt": [], "response": [], "system": [], "tools": [], "images": [], "videos": []}
     convert_images = partial(_convert_images, dataset_attr=dataset_attr, data_args=data_args)
+    convert_videos = partial(_convert_videos, dataset_attr=dataset_attr, data_args=data_args)
     tag_mapping = {
         dataset_attr.user_tag: Role.USER.value,
         dataset_attr.assistant_tag: Role.ASSISTANT.value,
@@ -185,6 +203,7 @@ def convert_sharegpt(
         outputs["system"].append(system)
         outputs["tools"].append(examples[dataset_attr.tools][i] if dataset_attr.tools else "")
         outputs["images"].append(convert_images(examples[dataset_attr.images][i]) if dataset_attr.images else [])
+        outputs["videos"].append(convert_videos(examples[dataset_attr.videos][i]) if dataset_attr.videos else [])
 
     return outputs
 
@@ -220,6 +239,7 @@ def align_dataset(
             "system": {"dtype": "string", "_type": "Value"},
             "tools": {"dtype": "string", "_type": "Value"},
             "images": [{"_type": "Image"}],
+            "videos": [{"dtype": "string", "_type": "Value"}],
         }
     )
     kwargs = {}
diff --git a/src/llamafactory/data/parser.py b/src/llamafactory/data/parser.py
index 2dccfc5d21..15a6eab8d3 100644
--- a/src/llamafactory/data/parser.py
+++ b/src/llamafactory/data/parser.py
@@ -43,6 +43,7 @@ class DatasetAttr:
     system: Optional[str] = None
     tools: Optional[str] = None
     images: Optional[str] = None
+    videos: Optional[str] = None
     # rlhf columns
     chosen: Optional[str] = None
     rejected: Optional[str] = None
@@ -126,7 +127,7 @@ def get_dataset_list(dataset_names: Optional[Sequence[str]], dataset_dir: str) -
         dataset_attr.set_attr("num_samples", dataset_info[name])
 
         if "columns" in dataset_info[name]:
-            column_names = ["system", "tools", "images", "chosen", "rejected", "kto_tag"]
+            column_names = ["system", "tools", "images", "videos", "chosen", "rejected", "kto_tag"]
             if dataset_attr.formatting == "alpaca":
                 column_names.extend(["prompt", "query", "response", "history"])
             else:
diff --git a/src/llamafactory/data/processors/feedback.py b/src/llamafactory/data/processors/feedback.py
index bed0c33c77..f030b35299 100644
--- a/src/llamafactory/data/processors/feedback.py
+++ b/src/llamafactory/data/processors/feedback.py
@@ -38,10 +38,17 @@ def _encode_feedback_example(
     template: "Template",
     tokenizer: "PreTrainedTokenizer",
     processor: Optional["ProcessorMixin"],
+    exists_images: bool,
+    exists_videos: bool,
     cutoff_len: int,
 ) -> Tuple[List[int], List[int], List[int], List[int], bool]:
-    if processor is not None and not hasattr(processor, "image_seq_length"):  # llava-like models
-        prompt[0]["content"] = template.image_token + prompt[0]["content"]
+    if processor is not None:
+        processor_class = type(processor).__name__
+        if processor_class != 'PaliGemmaProcessor':
+            if template.image_token not in prompt[0]["content"] and exists_videos:
+                prompt[0]["content"] = template.video_token + prompt[0]["content"]
+            if template.video_token not in prompt[0]["content"] and exists_images:
+                prompt[0]["content"] = template.image_token + prompt[0]["content"]
 
     if response[0]["content"]:  # desired example
         kto_tag = True
@@ -55,6 +62,21 @@ def _encode_feedback_example(
     else:
         kl_messages = prompt + [kl_response[1]]
 
+    if processor is not None and processor_class == 'Idefics2Processor':
+        fake_image_token = processor.fake_image_token.content
+        image_str = f"{fake_image_token}{template.image_token * processor.image_seq_len}{fake_image_token}"
+        image_str = image_str * 5
+        for j in range(len(messages)):
+            content = messages[j]['content']
+            content = content.replace(template.image_token, image_str)
+            content = content.replace(f"{fake_image_token}{fake_image_token}", f"{fake_image_token}")
+            messages[j]['content'] = content
+        for j in range(len(kl_messages)):
+            content = kl_messages[j]['content']
+            content = content.replace(template.image_token, image_str)
+            content = content.replace(f"{fake_image_token}{fake_image_token}", f"{fake_image_token}")
+            kl_messages[j]['content'] = content
+
     prompt_ids, response_ids = template.encode_oneturn(tokenizer, messages, system, tools)
     kl_prompt_ids, kl_response_ids = template.encode_oneturn(tokenizer, kl_messages, system, tools)
 
@@ -62,7 +84,7 @@ def _encode_feedback_example(
         response_ids += [tokenizer.eos_token_id]
         kl_response_ids += [tokenizer.eos_token_id]
 
-    if processor is not None and hasattr(processor, "image_seq_length"):  # paligemma models
+    if processor is not None and processor_class == 'PaliGemmaProcessor':  # paligemma models
         image_token_id = tokenizer.convert_tokens_to_ids(template.image_token)
         prompt_ids = [image_token_id] * getattr(processor, "image_seq_length") + prompt_ids
         kl_prompt_ids = [image_token_id] * getattr(processor, "image_seq_length") + kl_prompt_ids
@@ -120,6 +142,8 @@ def preprocess_feedback_dataset(
             template=template,
             tokenizer=tokenizer,
             processor=processor,
+            exists_images=len(examples["images"][i]) > 0,
+            exists_videos=len(examples['videos'][i]) > 0,
             cutoff_len=data_args.cutoff_len,
         )
         model_inputs["input_ids"].append(input_ids)
diff --git a/src/llamafactory/data/processors/pairwise.py b/src/llamafactory/data/processors/pairwise.py
index ddf885b51b..0907302f93 100644
--- a/src/llamafactory/data/processors/pairwise.py
+++ b/src/llamafactory/data/processors/pairwise.py
@@ -16,8 +16,7 @@
 
 from ...extras.constants import IGNORE_INDEX
 from ...extras.logging import get_logger
-from .processor_utils import get_paligemma_token_type_ids, get_pixel_values, infer_seqlen
-
+from .processor_utils import get_paligemma_token_type_ids, get_pixel_values, infer_seqlen, get_pixel_values_videos
 
 if TYPE_CHECKING:
     from transformers import PreTrainedTokenizer, ProcessorMixin
@@ -37,13 +36,36 @@ def _encode_pairwise_example(
     template: "Template",
     tokenizer: "PreTrainedTokenizer",
     processor: Optional["ProcessorMixin"],
+    exists_images: bool,
+    exists_videos: bool,
     cutoff_len: int,
 ) -> Tuple[List[int], List[int], List[int], List[int]]:
-    if processor is not None and not hasattr(processor, "image_seq_length"):  # llava-like models
-        prompt[0]["content"] = template.image_token + prompt[0]["content"]
+    if processor is not None:
+        processor_class = type(processor).__name__
+        if processor_class != 'PaliGemmaProcessor':
+            if template.image_token not in prompt[0]["content"] and exists_videos:
+                prompt[0]["content"] = template.video_token + prompt[0]["content"]
+            if template.video_token not in prompt[0]["content"] and exists_images:
+                prompt[0]["content"] = template.image_token + prompt[0]["content"]
 
     chosen_messages = prompt + [response[0]]
     rejected_messages = prompt + [response[1]]
+
+    if processor is not None and processor_class == 'Idefics2Processor':
+        fake_image_token = processor.fake_image_token.content
+        image_str = f"{fake_image_token}{template.image_token * processor.image_seq_len}{fake_image_token}"
+        image_str = image_str * 5
+        for j in range(len(chosen_messages)):
+            content = chosen_messages[j]['content']
+            content = content.replace(template.image_token, image_str)
+            content = content.replace(f"{fake_image_token}{fake_image_token}", f"{fake_image_token}")
+            chosen_messages[j]['content'] = content
+        for j in range(len(rejected_messages)):
+            content = rejected_messages[j]['content']
+            content = content.replace(template.image_token, image_str)
+            content = content.replace(f"{fake_image_token}{fake_image_token}", f"{fake_image_token}")
+            rejected_messages[j]['content'] = content
+
     prompt_ids, chosen_ids = template.encode_oneturn(tokenizer, chosen_messages, system, tools)
     _, rejected_ids = template.encode_oneturn(tokenizer, rejected_messages, system, tools)
 
@@ -51,7 +73,7 @@ def _encode_pairwise_example(
         chosen_ids += [tokenizer.eos_token_id]
         rejected_ids += [tokenizer.eos_token_id]
 
-    if processor is not None and hasattr(processor, "image_seq_length"):  # paligemma models
+    if processor is not None and processor_class == 'PaliGemmaProcessor':  # paligemma models
         image_token_id = tokenizer.convert_tokens_to_ids(template.image_token)
         prompt_ids = [image_token_id] * getattr(processor, "image_seq_length") + prompt_ids
 
@@ -85,9 +107,23 @@ def preprocess_pairwise_dataset(
         "rejected_attention_mask": [],
         "rejected_labels": [],
     }
+
+    image_keys = template.image_data_key
+    video_keys = template.video_data_key
+    processor_class = None
+
     if processor is not None:
-        model_inputs["pixel_values"] = []
-        if hasattr(processor, "image_seq_length"):  # paligemma models
+        if len(examples["images"][0]):
+            for image_key in image_keys:
+                model_inputs[image_key] = []
+
+        if len(examples["videos"][0]):
+            for video_key in video_keys:
+                model_inputs[video_key] = []
+
+        processor_class = type(processor).__name__
+
+        if processor_class == 'PaliGemmaProcessor':  # paligemma models
             model_inputs["chosen_token_type_ids"] = []
             model_inputs["rejected_token_type_ids"] = []
 
@@ -104,8 +140,11 @@ def preprocess_pairwise_dataset(
             template=template,
             tokenizer=tokenizer,
             processor=processor,
+            exists_images=len(examples["images"][i]) > 0,
+            exists_videos=len(examples['videos'][i]) > 0,
             cutoff_len=data_args.cutoff_len,
         )
+
         model_inputs["chosen_input_ids"].append(chosen_input_ids)
         model_inputs["chosen_attention_mask"].append([1] * len(chosen_input_ids))
         model_inputs["chosen_labels"].append(chosen_labels)
@@ -113,14 +152,26 @@ def preprocess_pairwise_dataset(
         model_inputs["rejected_attention_mask"].append([1] * len(rejected_input_ids))
         model_inputs["rejected_labels"].append(rejected_labels)
         if processor is not None:
-            model_inputs["pixel_values"].append(get_pixel_values(examples["images"][i], processor))
-            if hasattr(processor, "image_seq_length"):  # paligemma models
-                model_inputs["chosen_token_type_ids"].append(
-                    get_paligemma_token_type_ids(len(chosen_input_ids), processor)
-                )
-                model_inputs["rejected_token_type_ids"].append(
-                    get_paligemma_token_type_ids(len(rejected_input_ids), processor)
-                )
+            if len(examples["images"][i]):
+                image_data = get_pixel_values(examples["images"][i], processor, image_keys)
+                for image_key in image_keys:
+                    image_value = image_data[image_key]
+                    if image_value.shape[0] == 1:
+                        model_inputs[image_key].append(image_value[0])
+                if processor_class == 'PaliGemmaProcessor':  # paligemma models
+                    model_inputs["chosen_token_type_ids"].append(
+                        get_paligemma_token_type_ids(len(chosen_input_ids), processor)
+                    )
+                    model_inputs["rejected_token_type_ids"].append(
+                        get_paligemma_token_type_ids(len(rejected_input_ids), processor)
+                    )
+
+            if len(examples["videos"][i]):
+                video_data = get_pixel_values_videos(examples["videos"][i], processor, video_keys)
+                for video_key in video_keys:
+                    video_value = video_data[video_key]
+                    if video_value.shape[0] == 1:
+                        model_inputs[video_key].append(video_value[0])
 
     return model_inputs
 
diff --git a/src/llamafactory/data/processors/processor_utils.py b/src/llamafactory/data/processors/processor_utils.py
index 435cf6ca48..0fa38364a8 100644
--- a/src/llamafactory/data/processors/processor_utils.py
+++ b/src/llamafactory/data/processors/processor_utils.py
@@ -15,6 +15,9 @@
 import bisect
 from typing import TYPE_CHECKING, List, Sequence, Tuple
 
+import av
+import numpy as np
+
 from ...extras.packages import is_pillow_available
 
 
@@ -23,6 +26,7 @@
 
 
 if TYPE_CHECKING:
+    from av import Container
     from numpy.typing import NDArray
     from PIL.Image import Image as ImageObject
     from transformers import ProcessorMixin
@@ -61,13 +65,70 @@ def greedy_knapsack(numbers: List[int], capacity: int) -> List[List[int]]:
     return knapsacks
 
 
-def get_pixel_values(images: Sequence["ImageObject"], processor: "ProcessorMixin") -> "NDArray":
+def get_pixel_values(images: Sequence["ImageObject"], processor: "ProcessorMixin", image_keys: "list[str]" = ["pixel_values"]) -> "dict":
     r"""
     Processes visual inputs. (currently only supports a single image)
     """
     image_processor: "BaseImageProcessor" = getattr(processor, "image_processor")
-    image = images[0] if len(images) != 0 else Image.new("RGB", (100, 100), (255, 255, 255))
-    return image_processor(image, return_tensors="pt")["pixel_values"][0]  # shape (C, H, W)
+    if len(images) == 0:
+        image = Image.new("RGB", (100, 100), (255, 255, 255))
+        inputs = image_processor([image], return_tensors="pt")
+        return {k: inputs[k] for k in image_keys}
+    else:
+        inputs = image_processor(images, return_tensors="pt")
+        return {k: inputs[k] for k in image_keys}
+
+
+def preprocess_video(
+    video: "str"
+) -> "NDArray":
+    container = av.open(video)
+    total_frames = container.streams.video[0].frames
+    indices = np.arange(0, total_frames, total_frames / 8).astype(int)
+    clip = read_video_pyav(container, indices)
+    return clip
+
+
+def get_pixel_values_videos(
+    videos: Sequence["str"], processor: "ProcessorMixin", video_keys: "list[str]" = ["pixel_values_videos"]
+) -> "dict":
+    image_processor: "BaseImageProcessor" = getattr(processor, "image_processor")
+    video_processor = getattr(processor, "video_processor", None)
+    clips = []
+    for video in videos:
+        clip = preprocess_video(video)
+        clips.append(clip)
+    if len(clips) == 0:
+        clips = clips[0]
+    if video_processor is not None:
+        inputs = video_processor(clips, return_tensors="pt")
+        return {k: inputs[k] for k in video_keys}
+    else:
+        inputs = image_processor(videos=clips, padding=True, return_tensors="pt", images=None)
+        return {k: inputs[k] for k in video_keys}
+
+
+def read_video_pyav(container: "Container", indices: "NDArray"):
+    """
+    Decode the video with PyAV decoder.
+
+    Args:
+        container (av.container.input.InputContainer): PyAV container.
+        indices (List[int]): List of frame indices to decode.
+
+    Returns:
+        np.ndarray: np array of decoded frames of shape (num_frames, height, width, 3).
+    """
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format="rgb24") for x in frames])
 
 
 def get_paligemma_token_type_ids(input_len: int, processor: "ProcessorMixin") -> List[int]:
diff --git a/src/llamafactory/data/processors/supervised.py b/src/llamafactory/data/processors/supervised.py
index 950de12a86..f82eede5cc 100644
--- a/src/llamafactory/data/processors/supervised.py
+++ b/src/llamafactory/data/processors/supervised.py
@@ -17,7 +17,7 @@
 
 from ...extras.constants import IGNORE_INDEX
 from ...extras.logging import get_logger
-from .processor_utils import get_paligemma_token_type_ids, get_pixel_values, greedy_knapsack, infer_seqlen
+from .processor_utils import get_paligemma_token_type_ids, get_pixel_values, get_pixel_values_videos, greedy_knapsack, infer_seqlen
 
 
 if TYPE_CHECKING:
@@ -38,21 +38,38 @@ def _encode_supervised_example(
     template: "Template",
     tokenizer: "PreTrainedTokenizer",
     processor: Optional["ProcessorMixin"],
+    exists_images: bool,
+    exists_videos: bool,
     cutoff_len: int,
     train_on_prompt: bool,
     mask_history: bool,
 ) -> Tuple[List[int], List[int]]:
-    if processor is not None and not hasattr(processor, "image_seq_length"):  # llava-like models
-        prompt[0]["content"] = template.image_token + prompt[0]["content"]
+    if processor is not None:
+        processor_class = type(processor).__name__
+        if processor_class != 'PaliGemmaProcessor':
+            if template.image_token not in prompt[0]["content"] and exists_videos:
+                prompt[0]["content"] = template.video_token + prompt[0]["content"]
+            if template.video_token not in prompt[0]["content"] and exists_images:
+                prompt[0]["content"] = template.image_token + prompt[0]["content"]
 
     messages = prompt + response
     input_ids, labels = [], []
 
-    if processor is not None and hasattr(processor, "image_seq_length"):  # paligemma models
+    if processor is not None and processor_class == 'PaliGemmaProcessor':  # paligemma models
         image_token_id = tokenizer.convert_tokens_to_ids(template.image_token)
         input_ids += [image_token_id] * getattr(processor, "image_seq_length")
         labels += [IGNORE_INDEX] * getattr(processor, "image_seq_length")
 
+    if processor is not None and processor_class == 'Idefics2Processor':
+        fake_image_token = processor.fake_image_token.content
+        image_str = f"{fake_image_token}{template.image_token * processor.image_seq_len}{fake_image_token}"
+        image_str = image_str * 5
+        for j in range(len(messages)):
+            content = messages[j]['content']
+            content = content.replace(template.image_token, image_str)
+            content = content.replace(f"{fake_image_token}{fake_image_token}", f"{fake_image_token}")
+            messages[j]['content'] = content
+            
     encoded_pairs = template.encode_multiturn(tokenizer, messages, system, tools)
     total_length = 1 if template.efficient_eos else 0
     if mask_history:
@@ -103,9 +120,23 @@ def preprocess_supervised_dataset(
     # build inputs with format `<bos> X Y <eos>` and labels with format `<ignore> ... <ignore> Y <eos>`
     # for multiturn examples, we only mask the prompt part in each prompt-response pair.
     model_inputs = {"input_ids": [], "attention_mask": [], "labels": []}
+
+    image_keys = template.image_data_key
+    video_keys = template.video_data_key
+    processor_class = None
+
     if processor is not None:
-        model_inputs["pixel_values"] = []
-        if hasattr(processor, "image_seq_length"):  # paligemma models
+        if len(examples["images"][0]):
+            for image_key in image_keys:
+                model_inputs[image_key] = []
+
+        if len(examples["videos"][0]):
+            for video_key in video_keys:
+                model_inputs[video_key] = []
+
+        processor_class = type(processor).__name__
+
+        if processor_class == 'PaliGemmaProcessor':  # paligemma models
             model_inputs["token_type_ids"] = []
 
     for i in range(len(examples["prompt"])):
@@ -121,17 +152,32 @@ def preprocess_supervised_dataset(
             template=template,
             tokenizer=tokenizer,
             processor=processor,
+            exists_images=len(examples["images"][i]) > 0,
+            exists_videos=len(examples['videos'][i]) > 0,
             cutoff_len=data_args.cutoff_len,
             train_on_prompt=data_args.train_on_prompt,
             mask_history=data_args.mask_history,
         )
+
         model_inputs["input_ids"].append(input_ids)
         model_inputs["attention_mask"].append([1] * len(input_ids))
         model_inputs["labels"].append(labels)
         if processor is not None:
-            model_inputs["pixel_values"].append(get_pixel_values(examples["images"][i], processor))
-            if hasattr(processor, "image_seq_length"):  # paligemma models
-                model_inputs["token_type_ids"].append(get_paligemma_token_type_ids(len(input_ids), processor))
+            if len(examples["images"][i]):
+                image_data = get_pixel_values(examples["images"][i], processor, image_keys)
+                for image_key in image_keys:
+                    image_value = image_data[image_key]
+                    if image_value.shape[0] == 1:
+                        model_inputs[image_key].append(image_value[0])
+                if processor_class == 'PaliGemmaProcessor':  # paligemma models
+                    model_inputs["token_type_ids"].append(get_paligemma_token_type_ids(len(input_ids), processor))
+
+            if len(examples["videos"][i]):
+                video_data = get_pixel_values_videos(examples["videos"][i], processor, video_keys)
+                for video_key in video_keys:
+                    video_value = video_data[video_key]
+                    if video_value.shape[0] == 1:
+                        model_inputs[video_key].append(video_value[0])
 
     return model_inputs
 
@@ -161,6 +207,8 @@ def preprocess_packed_supervised_dataset(
             template=template,
             tokenizer=tokenizer,
             processor=None,
+            exists_images=len(examples["images"][i]) > 0,
+            exists_videos=len(examples["videos"][i]) > 0,
             cutoff_len=data_args.cutoff_len - 1,  # reserved for the padding token
             train_on_prompt=data_args.train_on_prompt,
             mask_history=data_args.mask_history,
diff --git a/src/llamafactory/data/processors/unsupervised.py b/src/llamafactory/data/processors/unsupervised.py
index 7bd1904b08..21ff9112c7 100644
--- a/src/llamafactory/data/processors/unsupervised.py
+++ b/src/llamafactory/data/processors/unsupervised.py
@@ -16,8 +16,7 @@
 
 from ...extras.logging import get_logger
 from ..data_utils import Role
-from .processor_utils import get_paligemma_token_type_ids, get_pixel_values, infer_seqlen
-
+from .processor_utils import get_paligemma_token_type_ids, get_pixel_values, infer_seqlen, get_pixel_values_videos
 
 if TYPE_CHECKING:
     from transformers import PreTrainedTokenizer, ProcessorMixin
@@ -37,21 +36,38 @@ def _encode_unsupervised_example(
     template: "Template",
     tokenizer: "PreTrainedTokenizer",
     processor: Optional["ProcessorMixin"],
+    exists_images: bool,
+    exists_videos: bool,
     cutoff_len: int,
 ) -> Tuple[List[int], List[int]]:
-    if processor is not None and not hasattr(processor, "image_seq_length"):  # llava-like models
-        prompt[0]["content"] = template.image_token + prompt[0]["content"]
+    if processor is not None:
+        processor_class = type(processor).__name__
+        if processor_class != 'PaliGemmaProcessor':
+            if template.image_token not in prompt[0]["content"] and exists_videos:
+                prompt[0]["content"] = template.video_token + prompt[0]["content"]
+            if template.video_token not in prompt[0]["content"] and exists_images:
+                prompt[0]["content"] = template.image_token + prompt[0]["content"]
 
     if len(response) == 1:
         messages = prompt + response
     else:
         messages = prompt + [{"role": Role.ASSISTANT.value, "content": ""}]
 
+    if processor is not None and processor_class == 'Idefics2Processor':
+        fake_image_token = processor.fake_image_token.content
+        image_str = f"{fake_image_token}{template.image_token * processor.image_seq_len}{fake_image_token}"
+        image_str = image_str * 5
+        for j in range(len(messages)):
+            content = messages[j]['content']
+            content = content.replace(template.image_token, image_str)
+            content = content.replace(f"{fake_image_token}{fake_image_token}", f"{fake_image_token}")
+            messages[j]['content'] = content
+
     input_ids, labels = template.encode_oneturn(tokenizer, messages, system, tools)
     if template.efficient_eos:
         labels += [tokenizer.eos_token_id]
 
-    if processor is not None and hasattr(processor, "image_seq_length"):  # paligemma models
+    if processor is not None and processor_class == 'PaliGemmaProcessor':  # paligemma models
         image_token_id = tokenizer.convert_tokens_to_ids(template.image_token)
         input_ids = [image_token_id] * getattr(processor, "image_seq_length") + input_ids
 
@@ -70,9 +86,23 @@ def preprocess_unsupervised_dataset(
 ) -> Dict[str, List[List[int]]]:
     # build inputs with format `<bos> X` and labels with format `Y <eos>`
     model_inputs = {"input_ids": [], "attention_mask": [], "labels": []}
+
+    image_keys = template.image_data_key
+    video_keys = template.video_data_key
+    processor_class = None
+
     if processor is not None:
-        model_inputs["pixel_values"] = []
-        if hasattr(processor, "image_seq_length"):  # paligemma models
+        if len(examples["images"][0]):
+            for image_key in image_keys:
+                model_inputs[image_key] = []
+
+        if len(examples["videos"][0]):
+            for video_key in video_keys:
+                model_inputs[video_key] = []
+
+        processor_class = type(processor).__name__
+
+        if processor_class == 'PaliGemmaProcessor':  # paligemma models
             model_inputs["token_type_ids"] = []
 
     for i in range(len(examples["prompt"])):
@@ -88,15 +118,30 @@ def preprocess_unsupervised_dataset(
             template=template,
             tokenizer=tokenizer,
             processor=processor,
+            exists_images=len(examples["images"][i]) > 0,
+            exists_videos=len(examples['videos'][i]) > 0,
             cutoff_len=data_args.cutoff_len,
         )
         model_inputs["input_ids"].append(input_ids)
         model_inputs["attention_mask"].append([1] * len(input_ids))
         model_inputs["labels"].append(labels)
+
         if processor is not None:
-            model_inputs["pixel_values"].append(get_pixel_values(examples["images"][i], processor))
-            if hasattr(processor, "image_seq_length"):  # paligemma models
-                model_inputs["token_type_ids"].append(get_paligemma_token_type_ids(len(input_ids), processor))
+            if len(examples["images"][i]):
+                image_data = get_pixel_values(examples["images"][i], processor, image_keys)
+                for image_key in image_keys:
+                    image_value = image_data[image_key]
+                    if image_value.shape[0] == 1:
+                        model_inputs[image_key].append(image_value[0])
+                if processor_class == 'PaliGemmaProcessor':  # paligemma models
+                    model_inputs["token_type_ids"].append(get_paligemma_token_type_ids(len(input_ids), processor))
+
+            if len(examples["videos"][i]):
+                video_data = get_pixel_values_videos(examples["videos"][i], processor, video_keys)
+                for video_key in video_keys:
+                    video_value = video_data[video_key]
+                    if video_value.shape[0] == 1:
+                        model_inputs[video_key].append(video_value[0])
 
     return model_inputs
 
diff --git a/src/llamafactory/data/template.py b/src/llamafactory/data/template.py
index f9eeb66a50..7ac1dae7e8 100644
--- a/src/llamafactory/data/template.py
+++ b/src/llamafactory/data/template.py
@@ -42,6 +42,9 @@ class Template:
     default_system: str
     stop_words: List[str]
     image_token: str
+    image_data_key: List[str]
+    video_token: str
+    video_data_key: List[str]
     efficient_eos: bool
     replace_eos: bool
 
@@ -206,6 +209,9 @@ def _register_template(
     default_system: str = "",
     stop_words: Sequence[str] = [],
     image_token: str = "<image>",
+    image_data_key: List[str] = ["pixel_values"],
+    video_token: str = "<video>",
+    video_data_key: List[str] = None,
     efficient_eos: bool = False,
     replace_eos: bool = False,
 ) -> None:
@@ -255,6 +261,9 @@ def _register_template(
         default_system=default_system,
         stop_words=stop_words,
         image_token=image_token,
+        image_data_key=image_data_key,
+        video_token=video_token,
+        video_data_key=video_data_key,
         efficient_eos=efficient_eos,
         replace_eos=replace_eos,
     )
@@ -654,6 +663,16 @@ def get_template_and_fix_tokenizer(
 )
 
 
+_register_template(
+    name="idefics2",
+    format_user=StringFormatter(slots=["User:{{content}}<end_of_utterance>\nAssistant:"]),
+    format_separator=EmptyFormatter(slots=["\n"]),
+    stop_words=["<end_of_utterance>"],
+    replace_eos=True,
+    image_data_key=["pixel_values", "pixel_attention_mask"],
+)
+
+
 _register_template(
     name="intern",
     format_user=StringFormatter(slots=["<|User|>:{{content}}\n<|Bot|>:"]),
@@ -716,6 +735,30 @@ def get_template_and_fix_tokenizer(
 )
 
 
+_register_template(
+    name="llava_next",
+    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
+    default_system=(
+        "A chat between a curious user and an artificial intelligence assistant. "
+        "The assistant gives helpful, detailed, and polite answers to the user's questions."
+    ),
+    image_data_key=["pixel_values", "image_sizes"],
+    image_token="<image>\n"
+)
+
+
+_register_template(
+    name="llava_next_video",
+    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
+    default_system=(
+        "A chat between a curious user and an artificial intelligence assistant. "
+        "The assistant gives helpful, detailed, and polite answers to the user's questions."
+    ),
+    image_data_key=["pixel_values", "image_sizes"],
+    video_data_key=["pixel_values_videos"],
+)
+
+
 _register_template(
     name="mistral",
     format_user=StringFormatter(slots=["[INST] {{content}} [/INST]"]),
@@ -834,6 +877,18 @@ def get_template_and_fix_tokenizer(
 )
 
 
+_register_template(
+    name="video_llava",
+    format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]),
+    default_system=(
+        "A chat between a curious user and an artificial intelligence assistant. "
+        "The assistant gives helpful, detailed, and polite answers to the user's questions."
+    ),
+    image_data_key=["pixel_values_images"],
+    video_data_key=["pixel_values_videos"],
+)
+
+
 _register_template(
     name="xuanyuan",
     format_user=StringFormatter(slots=["Human: {{content}} Assistant:"]),
diff --git a/src/llamafactory/extras/constants.py b/src/llamafactory/extras/constants.py
index 80df135cd0..710de58a29 100644
--- a/src/llamafactory/extras/constants.py
+++ b/src/llamafactory/extras/constants.py
@@ -579,6 +579,23 @@ def register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "Idefics2-Base": {
+            DownloadSource.DEFAULT: "HuggingFaceM4/idefics2-8b-base",
+        },
+        "Idefics2-Chat": {
+            DownloadSource.DEFAULT: "HuggingFaceM4/idefics2-8b",
+        },
+        "Idefics2-Chatty": {
+            DownloadSource.DEFAULT: "HuggingFaceM4/idefics2-8b-chatty",
+        },
+    },
+    template="idefics2",
+    vision=True,
+)
+
+
 register_model_group(
     models={
         "InternLM-7B": {
@@ -790,6 +807,49 @@ def register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "LLaVA-NeXT-7B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/llava-v1.6-vicuna-7b-hf",
+        },
+        "LLaVA-NeXT-13B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/llava-v1.6-vicuna-13b-hf",
+        },
+        "LLaVA-NeXT-34B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/llava-v1.6-34b-hf",
+        },
+        "LLaVA-NeXT-Mistral-7B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/llava-v1.6-mistral-7b-hf",
+        },
+    },
+    template="llava_next",
+    vision=True,
+)
+
+
+register_model_group(
+    models={
+        "LLaVA-NeXT-Video-7B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-7B-hf",
+        },
+        "LLaVA-NeXT-Video-34B-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-34B-hf",
+        },
+        "LLaVA-NeXT-Video-7B-32k-Chat": {
+            DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-7B-32K-hf",
+        },
+        "LLaVA-NeXT-Video-7B-DPO": {
+            DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-7B-DPO-hf",
+        },
+        "LLaVA-NeXT-Video-34B-DPO": {
+            DownloadSource.DEFAULT: "llava-hf/LLaVA-NeXT-Video-34B-DPO-hf",
+        },
+    },
+    template="llava_next_video",
+    vision=True,
+)
+
+
 register_model_group(
     models={
         "MiniCPM-2B-SFT-Chat": {
@@ -1409,6 +1469,17 @@ def register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "Video-LLaVA-7B-Chat": {
+            DownloadSource.DEFAULT: "LanguageBind/Video-LLaVA-7B-hf",
+        },
+    },
+    template="video_llava",
+    vision=True,
+)
+
+
 register_model_group(
     models={
         "XuanYuan-6B": {
diff --git a/src/llamafactory/hparams/data_args.py b/src/llamafactory/hparams/data_args.py
index 0cb4a56df0..a6faa1ce21 100644
--- a/src/llamafactory/hparams/data_args.py
+++ b/src/llamafactory/hparams/data_args.py
@@ -109,6 +109,14 @@ class DataArguments:
         default=None,
         metadata={"help": "Path to save or load the tokenized datasets."},
     )
+    image_key: Optional[str] = field(
+        default="pixel_values",
+        metadata={"help": "key of image pixel_values."},
+    )
+    video_key: Optional[str] = field(
+        default="pixel_values_videos",
+        metadata={"help": "key of video pixel_values."},
+    )
 
     def __post_init__(self):
         def split_arg(arg):
diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py
index fe700d5308..87e559d36d 100644
--- a/src/llamafactory/model/loader.py
+++ b/src/llamafactory/model/loader.py
@@ -114,6 +114,13 @@ def load_config(model_args: "ModelArguments") -> "PretrainedConfig":
     Loads model config.
     """
     init_kwargs = _get_init_kwargs(model_args)
+    if "LLaVA-NeXT-Video" in model_args.model_name_or_path:
+        from transformers import PretrainedConfig, LlavaNextVideoConfig, CLIPVisionConfig, LlamaConfig
+        official_config = PretrainedConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs)
+        config = LlavaNextVideoConfig(CLIPVisionConfig(**official_config.vision_config), LlamaConfig(**official_config.text_config))
+        setattr(config, "visual_inputs", True)
+        return config
+    
     return AutoConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs)
 
 
@@ -146,7 +153,11 @@ def load_model(
         if model_args.mixture_of_depths == "load":
             model = load_mod_pretrained_model(**init_kwargs)
         elif model_args.visual_inputs:
-            model = AutoModelForVision2Seq.from_pretrained(**init_kwargs)
+            if "llava_next_video" == getattr(config, "model_type"):
+                from transformers import LlavaNextVideoForConditionalGeneration
+                model = LlavaNextVideoForConditionalGeneration.from_pretrained(**init_kwargs)
+            else:
+                model = AutoModelForVision2Seq.from_pretrained(**init_kwargs)
         elif model_args.train_from_scratch:
             model = AutoModelForCausalLM.from_config(config)
         else:
diff --git a/src/llamafactory/model/model_utils/visual.py b/src/llamafactory/model/model_utils/visual.py
index 828a5e6d80..d88aa33409 100644
--- a/src/llamafactory/model/model_utils/visual.py
+++ b/src/llamafactory/model/model_utils/visual.py
@@ -95,7 +95,7 @@ def _mm_projector_forward_post_hook(
 
 
 def configure_visual_model(config: "PretrainedConfig") -> None:
-    if getattr(config, "model_type", None) == "llava":  # required for ds zero3 and valuehead models
+    if getattr(config, "model_type", None) in ["llava", "llava_next", "video_llava", "idefics2", "llava_next_video"]:  # required for ds zero3 and valuehead models
         setattr(config, "hidden_size", getattr(config.text_config, "hidden_size", None))
 
     if getattr(config, "is_yi_vl_derived_model", None):
diff --git a/src/llamafactory/webui/chatter.py b/src/llamafactory/webui/chatter.py
index 8abef92014..0953bc9178 100644
--- a/src/llamafactory/webui/chatter.py
+++ b/src/llamafactory/webui/chatter.py
@@ -136,6 +136,7 @@ def stream(
         system: str,
         tools: str,
         image: Optional[NDArray],
+        video: Optional[NDArray],
         max_new_tokens: int,
         top_p: float,
         temperature: float,
@@ -143,7 +144,7 @@ def stream(
         chatbot[-1][1] = ""
         response = ""
         for new_text in self.stream_chat(
-            messages, system, tools, image, max_new_tokens=max_new_tokens, top_p=top_p, temperature=temperature
+            messages, system, tools, image, video, max_new_tokens=max_new_tokens, top_p=top_p, temperature=temperature
         ):
             response += new_text
             if tools:
diff --git a/src/llamafactory/webui/components/chatbot.py b/src/llamafactory/webui/components/chatbot.py
index ad74114bae..415a95daae 100644
--- a/src/llamafactory/webui/components/chatbot.py
+++ b/src/llamafactory/webui/components/chatbot.py
@@ -46,6 +46,9 @@ def create_chat_box(
                     with gr.Column() as image_box:
                         image = gr.Image(sources=["upload"], type="numpy")
 
+                    with gr.Column() as video_box:
+                        video = gr.Video(sources=["upload"])
+
                 query = gr.Textbox(show_label=False, lines=8)
                 submit_btn = gr.Button(variant="primary")
 
@@ -63,7 +66,7 @@ def create_chat_box(
         [chatbot, messages, query],
     ).then(
         engine.chatter.stream,
-        [chatbot, messages, system, tools, image, max_new_tokens, top_p, temperature],
+        [chatbot, messages, system, tools, image, video, max_new_tokens, top_p, temperature],
         [chatbot, messages],
     )
     clear_btn.click(lambda: ([], []), outputs=[chatbot, messages])
@@ -77,7 +80,9 @@ def create_chat_box(
             system=system,
             tools=tools,
             image_box=image_box,
+            video_box=video_box,
             image=image,
+            video=video,
             query=query,
             submit_btn=submit_btn,
             max_new_tokens=max_new_tokens,
diff --git a/src/llamafactory/webui/components/infer.py b/src/llamafactory/webui/components/infer.py
index a006447994..e59c7d8ea8 100644
--- a/src/llamafactory/webui/components/infer.py
+++ b/src/llamafactory/webui/components/infer.py
@@ -70,4 +70,10 @@ def create_infer_tab(engine: "Engine") -> Dict[str, "Component"]:
         [chat_elems["image_box"]],
     )
 
+    engine.manager.get_elem_by_id("top.visual_inputs").change(
+        lambda enabled: gr.Column(visible=enabled),
+        [engine.manager.get_elem_by_id("top.visual_inputs")],
+        [chat_elems["video_box"]],
+    )
+
     return elem_dict