chore: Update dependencies to fix potential risks and improve compatibility

Author: Luodian

lmms_eval/models/llava_vid.py

@@ -17,30 +17,21 @@
 from lmms_eval.models.model_utils.load_video import read_video_pyav
 
 eval_logger = logging.getLogger("lmms-eval")
-import sys
 
-sys.path.append("llava-video")
 try:
-    from llavavid.model.language_model.llava_llama import LlavaConfig
-
-    # from llavavid.model.language_model.llava_qwen import LlavaQwenConfig
     from llavavid.model.builder import load_pretrained_model
     from llavavid.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
-    from llavavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+    from llavavid.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, IGNORE_INDEX
     from llavavid.conversation import conv_templates, SeparatorStyle
-
-    # AutoConfig.register("llava_qwen", LlavaQwenConfig)
-    AutoConfig.register("llava_llama", LlavaConfig)
-
+    from llavavid.mm_utils import tokenizer_image_token_qwen_merge, preprocess_qwen, preprocess_llama3
 except ImportError:
     eval_logger.debug("LLaVA-Video is not installed. Please install LLaVA-Video to use this model.")
 
-try:
-    from llavavid.model.language_model.llava_qwen import LlavaQwenConfig
+from llavavid.model.language_model.llava_qwen import LlavaQwenConfig
+from llavavid.model.language_model.llava_llama import LlavaConfig
 
-    AutoConfig.register("llava_qwen", LlavaQwenConfig)
-except:
-    eval_logger.debug("")
+AutoConfig.register("llava_qwen", LlavaQwenConfig)
+AutoConfig.register("llava_llama", LlavaConfig)
 
 
 @register_model("llavavid")
@@ -96,7 +87,6 @@ def __init__(
         self.mm_spatial_pool_out_channels = int(mm_spatial_pool_out_channels)
         self.mm_spatial_pool_mode = mm_spatial_pool_mode
         self.max_frames_num = int(max_frames_num)
-        print(self.max_frames_num)
         if self.overwrite == True:
             overwrite_config = {}
             overwrite_config["mm_resampler_type"] = self.mm_resampler_type
@@ -361,7 +351,7 @@ def generate_until(self, requests) -> List[str]:
             if self.model.config.mm_use_im_start_end:
                 qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + qs
             else:
-                qs = DEFAULT_IMAGE_TOKEN + "\n" + qs
+                qs = DEFAULT_IMAGE_TOKEN * len(videos) + "\n" + qs
 
             # This is much safer for llama3, as we now have some object type in it
             if "llama_3" in self.conv_template:
@@ -379,11 +369,6 @@ def generate_until(self, requests) -> List[str]:
                 pad_token_ids = 0  # lmms-lab/llama3-llava-8b is trained on this pad token id. You may need to customize this for other models.
             attention_masks = input_ids.ne(pad_token_ids).long().cuda()
 
-            # input_ids_list = [tokenizer_image_token(prompt, self.tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt") for prompt in question_input]
-            # pad_token_ids = self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id
-            # input_ids = self.pad_sequence(input_ids_list, batch_first=True, padding_value=pad_token_ids).to(self.device)
-            # attention_masks = input_ids.ne(pad_token_ids).to(self.device)
-
             stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
             keywords = [stop_str]
             stopping_criteria = KeywordsStoppingCriteria(keywords, self.tokenizer, input_ids)
@@ -393,7 +378,7 @@ def generate_until(self, requests) -> List[str]:
             if "max_new_tokens" not in gen_kwargs:
                 gen_kwargs["max_new_tokens"] = 1024
             if "temperature" not in gen_kwargs:
-                gen_kwargs["temperature"] = 0.2
+                gen_kwargs["temperature"] = 0
             if "top_p" not in gen_kwargs:
                 gen_kwargs["top_p"] = None
             if "num_beams" not in gen_kwargs:
@@ -417,4 +402,4 @@ def generate_until(self, requests) -> List[str]:
             outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
             res.append(outputs)
             pbar.update(1)
-        return res
+        return res

lmms_eval/tasks/mmmu/utils.py

@@ -86,8 +86,9 @@ def extract_subset_name(input_string):
 
 def mmmu_test_aggregate_results_for_submission(results, args):
     path = generate_submission_file("mmmu_test_for_submission.json", args)
+    results_dict = {list(item.keys())[0]: list(item.values())[0] for item in results}
     with open(path, "w") as f:
-        json.dump(results, f)
+        json.dump(results_dict, f)
     lmms_logger.info(f"Results saved to {path}.")
 
 

pyproject.toml

@@ -28,6 +28,7 @@ dependencies = [
     "httpx==0.23.3",
     "jsonlines",
     "numexpr",
+    "numpy==1.26.4",
     "peft>=0.2.0",
     "pybind11>=2.6.2",
     "pytablewriter",
@@ -52,7 +53,7 @@ dependencies = [
     "openpyxl",
     "Levenshtein",
     "hf_transfer",
-    "tenacity",
+    "tenacity==8.3.0",
     "wandb>=0.16.0",
     "tiktoken",
     "pre-commit",