EvolvingLMMs-Lab
diff --git a/‎lmms_eval/models/__init__.py‎
Lines changed: 8 additions & 12 deletions b/‎lmms_eval/models/__init__.py‎
Lines changed: 8 additions & 12 deletions
diff --git a/‎lmms_eval/models/internvl2.py‎
Lines changed: 1 addition & 1 deletion b/‎lmms_eval/models/internvl2.py‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎lmms_eval/models/llava_onevision.py‎
Lines changed: 0 additions & 9 deletions b/‎lmms_eval/models/llava_onevision.py‎
Lines changed: 0 additions & 9 deletions
diff --git a/‎lmms_eval/models/vila.py‎
Lines changed: 2 additions & 15 deletions b/‎lmms_eval/models/vila.py‎
Lines changed: 2 additions & 15 deletions
diff --git a/‎lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml‎
Lines changed: 1 addition & 1 deletion b/‎lmms_eval/tasks/llava_interleave_bench/_default_template_interleave_yaml‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎lmms_eval/tasks/llava_interleave_bench/utils.py‎
Lines changed: 24 additions & 0 deletions b/‎lmms_eval/tasks/llava_interleave_bench/utils.py‎
Lines changed: 24 additions & 0 deletions
diff --git a/‎lmms_eval/tasks/mathvista/mathvista.yaml‎
Lines changed: 1 addition & 1 deletion b/‎lmms_eval/tasks/mathvista/mathvista.yaml‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎lmms_eval/tasks/mathvista/mathvista_test.yaml‎
Lines changed: 1 addition & 6 deletions b/‎lmms_eval/tasks/mathvista/mathvista_test.yaml‎
Lines changed: 1 addition & 6 deletions
diff --git a/‎lmms_eval/tasks/mathvista/mathvista_testmini.yaml‎
Lines changed: 9 additions & 34 deletions b/‎lmms_eval/tasks/mathvista/mathvista_testmini.yaml‎
Lines changed: 9 additions & 34 deletions
diff --git a/‎lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml‎
Lines changed: 29 additions & 0 deletions b/‎lmms_eval/tasks/mathvista/mathvista_testmini_cot.yaml‎
Lines changed: 29 additions & 0 deletions
@@ -3,6 +3,9 @@
 import hf_transfer
 from loguru import logger
 import sys
+import hf_transfer
+
+os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
 
 logger.remove()
 logger.add(sys.stdout, level="WARNING")
@@ -33,19 +36,16 @@
     "phi3v": "Phi3v",
     "tinyllava": "TinyLlava",
     "llava_hf": "LlavaHf",
-    "longva": "LongVA",
-    "llava_onevision": "Llava_OneVision",
-    "llava_hf": "LlavaHf",
+    "llava_onevision": "LlavaOneVision",
     "longva": "LongVA",
     "vila": "VILA",
 }
 
 for model_name, model_class in AVAILABLE_MODELS.items():
     try:
         exec(f"from .{model_name} import {model_class}")
-    except ImportError as e:
-        # logger.warning(f"Failed to import {model_class} from {model_name}: {e}")
-        pass
+    except Exception as e:
+        logger.debug(f"Failed to import {model_class} from {model_name}: {e}")
 
 if os.environ.get("LMMS_EVAL_PLUGINS", None):
     # Allow specifying other packages to import models from
@@ -54,9 +54,5 @@
         for model_name, model_class in getattr(m, "AVAILABLE_MODELS").items():
             try:
                 exec(f"from {plugin}.models.{model_name} import {model_class}")
-            except ImportError:
-                pass
-
-import hf_transfer
-
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+            except ImportError as e:
+                logger.debug(f"Failed to import {model_class} from {model_name}: {e}")
@@ -213,7 +213,7 @@ def generate_until(self, requests) -> List[str]:
             for k, v in gen_kwargs.items():
                 if k not in DEFAULT_GEN_KWARGS:
                     pop_keys.append(k)
-            
+
             for k in pop_keys:
                 gen_kwargs.pop(k)
 
 
@@ -54,15 +54,6 @@
 except ImportError as e:
     eval_logger.debug(f"LLaVA is not installed. Please install LLaVA to use this model.\nError: {e}")
 
-# Import LLaVA-vid modules
-try:
-    from llavavid.model.language_model.llava_qwen import LlavaQwenConfig
-    from llavavid.model.language_model.llava_llama import LlavaConfig
-
-    AutoConfig.register("llava_qwen", LlavaQwenConfig)
-    AutoConfig.register("llava_llama", LlavaConfig)
-except ImportError as e:
-    eval_logger.debug(f"LLaVA-vid is not installed. Error: {e}")
 
 # Determine best attention implementation
 if version.parse(torch.__version__) >= version.parse("2.1.2"):
 
@@ -33,9 +33,7 @@
     from llava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
     from llava.mm_utils import process_images
 except ImportError as e:
-    print(e)
-    # import pdb;pdb.set_trace()
-    eval_logger.debug("VILA is not installed. Please install VILA to use this model.")
+    eval_logger.debug(f"VILA is not installed. Please install VILA to use this model. Error: {e}")
 
 
 @register_model("vila")
@@ -81,7 +79,6 @@ def __init__(
         self.max_frames_num = max_frames_num
         # self._config = AutoConfig.from_pretrained(self.pretrained)
 
-        # import pdb; pdb.set_trace()
         self._tokenizer, self._model, self._image_processor, self._max_length = load_pretrained_model(pretrained, self.model_name, device_map=self.device_map, attn_implementation=attn_implementation)
 
         self.model.image_processor = self._image_processor
@@ -202,7 +199,6 @@ def load_video(self, video_path, max_frames_num):
             return [Image.fromarray(img) for img in spare_frames]
         except Exception as e:
             eval_logger.error(f"Failed to load video {video_path} with error: {e}")
-            # import pdb;pdb.set_trace()
             return [Image.new("RGB", (448, 448), (0, 0, 0))] * max_frames_num
 
     def tok_decode(self, tokens):
@@ -278,31 +274,23 @@ def generate_until(self, requests) -> List[str]:
         pbar = tqdm(total=len(requests), disable=(self.rank != 0), desc="Model Responding")
 
         for contexts, gen_kwargs, doc_to_visual, doc_id, task, split in [reg.args for reg in requests]:
-            # if self.task_dict[task][split][doc_id]["duration"] != "short":
-            #     # import pdb;pdb.set_trace()
-            #     res.append("A")
-            #     pbar.update(1)
-            #     continue
             # encode, pad, and truncate contexts for this batch
             visuals = [doc_to_visual(self.task_dict[task][split][doc_id])]
             visuals = self.flatten(visuals)
 
             num_video_frames = self.model.config.num_video_frames
             videos = []
-            # import pdb;pdb.set_trace()
             if self.max_frames_num == 0:
                 images = [Image.new("RGB", (448, 448), (0, 0, 0))] * num_video_frames
                 video = process_images(images, self.model.image_processor, self.model.config).half().cuda()
                 videos.append(video)
             else:
                 for visual in visuals:
-                    # images, video_loading_succeed = LazySupervisedDataset._load_video(visual, num_video_frames, self.model)
-                    # import pdb;pdb.set_trace()
                     if self.video_decode_backend == "decord":
                         images = self.load_video(visual, num_video_frames)
                     elif self.video_decode_backend == "pyav":
                         images = read_video_pyav(visual, num_frm=num_video_frames)
-                    # import pdb;pdb.set_trace()
+
                     video = process_images(images, self.model.image_processor, self.model.config).half().cuda()
                     videos.append(video)
 
@@ -370,7 +358,6 @@ def generate_until(self, requests) -> List[str]:
             outputs = self.tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
             print("Question: ", cur_prompt)
             print("Answer: ", outputs)
-            # import pdb;pdb.set_trace()
             res.append(outputs)
             pbar.update(1)
         return res
@@ -2,7 +2,7 @@ output_type: generate_until
 generation_kwargs:
   until:
     - "ASSISTANT:"
-  image_aspect_ratio: original
+  image_aspect_ratio: pad
 metadata:
   version: 0.0
   api_type : openai
 
@@ -22,6 +22,16 @@
 nlrv2 = ["NLVR2_Mantis"]
 qbench = ["QBench"]
 
+scan_qa = ["ScanQA"]
+alfred = ["ALFRED"]
+nuscenes = ["nuscenes"]
+scannet_chat = ["ScanNet_chat"]
+scannet_task = ["ScanNet_task"]
+blink = ["BLINK"]
+math_verse = ["MathVerse"]
+sci_verse = ["SciVerse"]
+mantis = ["Mantis"]
+
 
 def doc_to_visual(doc):
     max_visual_count = 16
@@ -184,9 +194,19 @@ def overall_score(results):
         "Puzzle": puzzle,
         "NLVR2": nlrv2,
         "QBench": qbench,
+        "ScanQA": scan_qa,
+        "ALFRED": alfred,
+        "nuscenes": nuscenes,
+        "ScanNet_chat": scannet_chat,
+        "ScanNet_task": scannet_task,
+        "BLINK": blink,
+        "MathVerse": math_verse,
+        "SciVerse": sci_verse,
+        "Mantis": mantis,
     }
 
     category_scores = {}
+    matched_subtasks = set()
 
     eval_logger.info(f"Evaluation Sub-Task Results:")
     for category, subtasks in categories.items():
@@ -196,11 +216,15 @@ def overall_score(results):
             if result["sub_task"] in subtasks:
                 count += 1
                 score += result["score"]
+                matched_subtasks.add(result["sub_task"])
         if count > 0:
             avg_score = score / count
             category_scores[category] = avg_score
             eval_logger.info(f"{category}: {avg_score:.3f}")
 
+    if not matched_subtasks:
+        raise ValueError("No subtasks were matched in the results. Check if the subtask names are correct.")
+
     # Calculate overall score
     total_score = sum(category_scores.values())
     num_categories = len(category_scores)
 
@@ -4,5 +4,5 @@ task:
   - mathvista_test
 metadata:
   version: 0.0
-  gpt_eval_model_name: "gpt-4-0613"
+  gpt_eval_model_name: "gpt-3.5-turbo"
   quick_extract: false
@@ -8,8 +8,6 @@ doc_to_visual: !function utils.mathvista_doc_to_visual
 doc_to_text: !function utils.mathvista_doc_to_text
 doc_to_target: "answer"
 generation_kwargs:
-  until:
-    - "ASSISTANT:"
   max_new_tokens: 1024
   temperature: 0
   top_p: 1.0
@@ -23,7 +21,4 @@ metric_list:
 
 model_specific_prompt_kwargs:
   default:
-    shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step"
-model_specific_generation_kwargs:
-  llava:
-    image_aspect_ratio: original
+    shot_type: "reason-first" # can be "reason-first", "solution", "step-by-step"
@@ -1,34 +1,9 @@
-dataset_path: AI4Math/MathVista
-dataset_kwargs:
-  token: True
-task: "mathvista_testmini"
-test_split: testmini
-output_type: generate_until
-doc_to_visual: !function utils.mathvista_doc_to_visual
-doc_to_text: !function utils.mathvista_doc_to_text
-doc_to_target: "answer"
-generation_kwargs:
-  until:
-    - "ASSISTANT:"
-  max_new_tokens: 1024
-  temperature: 0
-  top_p: 1.0
-  num_beams: 1
-  do_sample: false
-process_results: !function utils.mathvista_process_results
-metric_list:
-  - metric: gpt_eval_score
-    aggregation: !function utils.mathvista_aggregate_results
-    higher_is_better: true
-
-model_specific_prompt_kwargs:
-  default:
-    shot_type: "format-prompt" # can be "reason-first", "solution", "step-by-step"
-    shot: 0
-    use_caption: False
-    use_ocr: False
-  phi3v:
-    shot_type: "solution"
-model_specific_generation_kwargs:
-  llava:
-    image_aspect_ratio: original
+group: mathvista_testmini
+task:
+  - mathvista_testmini_cot
+  - mathvista_testmini_solution
+  - mathvista_testmini_format
+metadata:
+  version: 0.0
+  gpt_eval_model_name: "gpt-3.5-turbo"
+  quick_extract: false
@@ -0,0 +1,29 @@
+dataset_path: AI4Math/MathVista
+dataset_kwargs:
+  token: True
+task: "mathvista_testmini_cot"
+test_split: testmini
+output_type: generate_until
+doc_to_visual: !function utils.mathvista_doc_to_visual
+doc_to_text: !function utils.mathvista_doc_to_text
+doc_to_target: "answer"
+generation_kwargs:
+  max_new_tokens: 1024
+  temperature: 0
+  top_p: 1.0
+  num_beams: 1
+  do_sample: false
+process_results: !function utils.mathvista_process_results
+metric_list:
+  - metric: gpt_eval_score
+    aggregation: !function utils.mathvista_aggregate_results
+    higher_is_better: true
+
+model_specific_prompt_kwargs:
+  default:
+    shot_type: "step-by-step" # can be "reason-first", "solution", "step-by-step"
+    shot: 0
+    use_caption: False
+    use_ocr: False
+  phi3v:
+    shot_type: "solution"