From 5340a30d0193547a19e236757fec1f3f246642f9 Mon Sep 17 00:00:00 2001 From: Alex Brooks Date: Mon, 13 Jan 2025 01:37:48 -0700 Subject: [PATCH] Fix Max Token ID for Qwen-VL-Chat (#11980) Signed-off-by: Alex-Brooks --- vllm/transformers_utils/tokenizer.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 97920f42ec52f..294262484f2fb 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -1,3 +1,4 @@ +import contextlib import os import warnings from pathlib import Path @@ -67,7 +68,15 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer: tokenizer.all_special_tokens_extended) tokenizer_all_special_tokens = set(tokenizer.all_special_tokens) tokenizer_len = len(tokenizer) + max_token_id = max(tokenizer.get_vocab().values()) + # Some tokenizers (e.g., QwenTokenizer) have special tokens that + # are added and included in the implementation of the vocab_size + # property, but not in get_vocab(); if there is an implementation + # of vocab size, we should take the greater value. + if hasattr(tokenizer, "vocab_size"): + with contextlib.suppress(NotImplementedError): + max_token_id = max(max_token_id, tokenizer.vocab_size) class CachedTokenizer(tokenizer.__class__): # type: ignore