From 8b7c570848f758d41a2cc5fc3d67c058fa10524d Mon Sep 17 00:00:00 2001 From: "r.4ntix" Date: Mon, 17 Feb 2025 15:26:50 +0800 Subject: [PATCH 1/2] [Bugfix] Fix VLLM_USE_MODELSCOPE issue * When using VLLM_USE_MODELSCOPE, the huggingface_hub API will be used to get the model file list. --- vllm/transformers_utils/config.py | 18 ++++++++++++------ vllm/transformers_utils/utils.py | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 4b76509e4541..a32ae7ca0138 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -117,6 +117,11 @@ def list_repo_files( def lookup_files(): try: + if VLLM_USE_MODELSCOPE: + from vllm.transformers_utils.utils import modelscope_list_repo_files + return modelscope_list_repo_files(repo_id, + revision=revision, + token=token) return hf_list_repo_files(repo_id, revision=revision, repo_type=repo_type, @@ -382,17 +387,17 @@ def get_hf_file_to_dict(file_name: str, @cache def get_pooling_config(model: str, revision: Optional[str] = 'main'): """ - This function gets the pooling and normalize - config from the model - only applies to - sentence-transformers models. + This function gets the pooling and normalize + config from the model - only applies to + sentence-transformers models. Args: model (str): The name of the Hugging Face model. - revision (str, optional): The specific version + revision (str, optional): The specific version of the model to use. Defaults to 'main'. Returns: - dict: A dictionary containing the pooling + dict: A dictionary containing the pooling type and whether normalization is used. """ @@ -498,8 +503,9 @@ def get_sentence_transformer_tokenizer_config(model: str, repo_files = list_repo_files(model, revision=revision, token=HF_TOKEN) + logger.info(repo_files) except Exception as e: - logger.debug("Error getting repo files", e) + logger.error("Error getting repo files", e) repo_files = [] for config_name in sentence_transformer_config_files: diff --git a/vllm/transformers_utils/utils.py b/vllm/transformers_utils/utils.py index 71fe3ef0b23c..beb1699c9456 100644 --- a/vllm/transformers_utils/utils.py +++ b/vllm/transformers_utils/utils.py @@ -2,7 +2,7 @@ from os import PathLike from pathlib import Path -from typing import Union +from typing import List, Optional, Union def is_s3(model_or_path: str) -> bool: @@ -20,3 +20,17 @@ def check_gguf_file(model: Union[str, PathLike]) -> bool: with open(model, "rb") as f: header = f.read(4) return header == b"GGUF" + +def modelscope_list_repo_files( + repo_id: str, + revision: Optional[str] = None, + token: Union[str, bool, None] = None, + ) -> List[str]: + """List files in a modelscope repo.""" + from modelscope.utils.hf_util import _try_login + from modelscope.hub.api import HubApi + _try_login(token) + api = HubApi() + # same as huggingface_hub.list_repo_files + files = [ file['Path'] for file in api.get_model_files(model_id=repo_id, revision=revision, recursive=True) if file['Type'] == 'blob'] + return files From a0d07eb3b5e42cd5f0edffc94df554643f690302 Mon Sep 17 00:00:00 2001 From: "r.4ntix" Date: Mon, 17 Feb 2025 20:15:06 +0800 Subject: [PATCH 2/2] Fix pre-commit errors --- vllm/transformers_utils/config.py | 8 ++++---- vllm/transformers_utils/utils.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index a32ae7ca0138..360b457a19a8 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -118,10 +118,11 @@ def list_repo_files( def lookup_files(): try: if VLLM_USE_MODELSCOPE: - from vllm.transformers_utils.utils import modelscope_list_repo_files + from vllm.transformers_utils.utils import ( + modelscope_list_repo_files) return modelscope_list_repo_files(repo_id, - revision=revision, - token=token) + revision=revision, + token=token) return hf_list_repo_files(repo_id, revision=revision, repo_type=repo_type, @@ -503,7 +504,6 @@ def get_sentence_transformer_tokenizer_config(model: str, repo_files = list_repo_files(model, revision=revision, token=HF_TOKEN) - logger.info(repo_files) except Exception as e: logger.error("Error getting repo files", e) repo_files = [] diff --git a/vllm/transformers_utils/utils.py b/vllm/transformers_utils/utils.py index beb1699c9456..d0b5d7f01a99 100644 --- a/vllm/transformers_utils/utils.py +++ b/vllm/transformers_utils/utils.py @@ -21,16 +21,21 @@ def check_gguf_file(model: Union[str, PathLike]) -> bool: header = f.read(4) return header == b"GGUF" + def modelscope_list_repo_files( - repo_id: str, - revision: Optional[str] = None, - token: Union[str, bool, None] = None, - ) -> List[str]: + repo_id: str, + revision: Optional[str] = None, + token: Union[str, bool, None] = None, +) -> List[str]: """List files in a modelscope repo.""" - from modelscope.utils.hf_util import _try_login from modelscope.hub.api import HubApi + from modelscope.utils.hf_util import _try_login _try_login(token) api = HubApi() # same as huggingface_hub.list_repo_files - files = [ file['Path'] for file in api.get_model_files(model_id=repo_id, revision=revision, recursive=True) if file['Type'] == 'blob'] + files = [ + file['Path'] for file in api.get_model_files( + model_id=repo_id, revision=revision, recursive=True) + if file['Type'] == 'blob' + ] return files