EvolvingLMMs-Lab
diff --git a/‎LICENSE‎
Lines changed: 56 additions & 0 deletions b/‎LICENSE‎
Lines changed: 56 additions & 0 deletions
diff --git a/‎README.md‎
Lines changed: 171 additions & 195 deletions b/‎README.md‎
Lines changed: 171 additions & 195 deletions
diff --git a/‎docs/current_tasks.md‎
Lines changed: 122 additions & 0 deletions b/‎docs/current_tasks.md‎
Lines changed: 122 additions & 0 deletions
diff --git a/‎lmms_eval/api/task.py‎
Lines changed: 42 additions & 0 deletions b/‎lmms_eval/api/task.py‎
Lines changed: 42 additions & 0 deletions
diff --git a/‎lmms_eval/models/__init__.py‎
Lines changed: 3 additions & 1 deletion b/‎lmms_eval/models/__init__.py‎
Lines changed: 3 additions & 1 deletion
@@ -0,0 +1,56 @@
+# For the main pipeline structure-related code, we maintain the original license provided with lm-evaluation-harness, which is the MIT License.
+
+MIT License
+
+Copyright (c) 2024 LMMs-Lab
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+# For the multimodal models and datasets that we have added (defined as code in the lmms_eval/tasks and lmms_eval/models folders), we apply the Apache License.
+
+Apache 2.0 License
+
+Copyright (c) 2024 LMMs-Lab
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+When modifying the code, please include the following information about the original lmms-eval source:
+# Adopted from lmms-eval from https://github.com/EvolvingLMMs-Lab/lmms-eval. Below is the original copyright:
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
@@ -0,0 +1,122 @@
+# Current Tasks
+
+> () indicates the task name in the lmms_eval. The task name is also used to specify the dataset in the configuration file.
+> The following is manually updated documentation. You could use `lmms_eval task --list` to list all supported tasks and their task names. 
+
+- AI2D (ai2d)
+- ChartQA (chartqa)
+- CMMMU (cmmmu)
+  - CMMMU Validation (cmmmu_val)
+  - CMMMU Test (cmmmu_test)
+- COCO Caption (coco_cap)
+  - COCO 2014 Caption (coco2014_cap)
+    - COCO 2014 Caption Validation (coco2014_cap_val)
+    - COCO 2014 Caption Test (coco2014_cap_test)
+  - COCO 2017 Caption (coco2017_cap)
+    - COCO 2017 Caption MiniVal (coco2017_cap_val)
+    - COCO 2017 Caption MiniTest (coco2017_cap_test)
+- [ConBench](https://github.com/foundation-multimodal-models/ConBench) (conbench)
+- DOCVQA (docvqa)
+  - DOCVQA Validation (docvqa_val)
+  - DOCVQA Test (docvqa_test)
+- Ferret (ferret)
+- Flickr30K (flickr30k)
+  - Ferret Test (ferret_test)
+- GQA (gqa)
+- HallusionBenchmark (hallusion_bench_image)
+- Infographic VQA (info_vqa)
+  - Infographic VQA Validation (info_vqa_val)
+  - Infographic VQA Test (info_vqa_test)
+- LLaVA-Bench (llava_in_the_wild)
+- LLaVA-Bench-COCO (llava_bench_coco)
+- MathVerse (mathverse)
+  - MathVerse Text Dominant (mathverse_testmini_text_dominant)
+  - MathVerse Text Only (mathverse_testmini_text_only)
+  - MathVerse Text Lite (mathverse_testmini_text_lite)
+  - MathVerse Vision Dominant (mathverse_testmini_vision_dominant)
+  - MathVerse Vision Intensive (mathverse_testmini_vision_intensive)
+  - MathVerse Vision Only (mathverse_testmini_vision_only)
+- MathVista (mathvista)
+  - MathVista Validation (mathvista_testmini)
+  - MathVista Test (mathvista_test)
+- MMBench (mmbench)
+  - MMBench English (mmbench_en)
+    - MMBench English Dev (mmbench_en_dev)
+    - MMBench English Test (mmbench_en_test)
+  - MMBench Chinese (mmbench_cn)
+    - MMBench Chinese Dev (mmbench_cn_dev)
+    - MMBench Chinese Test (mmbench_cn_test)
+- MME (mme)
+- MMMU (mmmu)
+  - MMMU Validation (mmmu_val)
+  - MMMU Test (mmmu_test)
+- MMUPD (mmupd)
+  - MMUPD Base (mmupd_base)
+    - MMAAD Base (mmaad_base)
+    - MMIASD Base (mmiasd_base)
+    - MMIVQD Base (mmivqd_base)
+  - MMUPD Option (mmupd_option)
+    - MMAAD Option (mmaad_option)
+    - MMIASD Option (mmiasd_option)
+    - MMIVQD Option (mmivqd_option)
+  - MMUPD Instruction (mmupd_instruction)
+    - MMAAD Instruction (mmaad_instruction)
+    - MMIASD Instruction (mmiasd_instruction)
+    - MMIVQD Instruction (mmivqd_instruction)
+- MMVet (mmvet)
+- Multi-DocVQA (multidocvqa)
+  - Multi-DocVQA Validation (multidocvqa_val)
+  - Multi-DocVQA Test (multidocvqa_test)
+- NoCaps (nocaps)
+  - NoCaps Validation (nocaps_val)
+  - NoCaps Test (nocaps_test)
+- OKVQA (ok_vqa)
+  - OKVQA Validation 2014 (ok_vqa_val2014)
+- POPE (pope)
+- RefCOCO (refcoco)
+    - refcoco_seg_test
+    - refcoco_seg_val
+    - refcoco_seg_testA
+    - refcoco_seg_testB
+    - refcoco_bbox_test
+    - refcoco_bbox_val
+    - refcoco_bbox_testA
+    - refcoco_bbox_testB
+- RefCOCO+ (refcoco+)
+    - refcoco+_seg
+        - refcoco+_seg_val
+        - refcoco+_seg_testA
+        - refcoco+_seg_testB
+    - refcoco+_bbox
+        - refcoco+_bbox_val
+        - refcoco+_bbox_testA
+        - refcoco+_bbox_testB
+- RefCOCOg (refcocog)
+    - refcocog_seg_test
+    - refcocog_seg_val
+    - refcocog_bbox_test
+    - refcocog_bbox_val
+- ScienceQA (scienceqa_full)
+  - ScienceQA Full (scienceqa)
+  - ScienceQA IMG (scienceqa_img)
+- ScreenSpot (screenspot)
+  - ScreenSpot REC / Grounding (screenspot_rec)
+  - ScreenSpot REG / Instruction Generation (screenspot_reg)
+- SeedBench (seedbench)
+- SeedBench 2 (seedbench_2)
+- ST-VQA (stvqa)
+- TextCaps (textcaps)
+  - TextCaps Validation (textcaps_val)
+  - TextCaps Test (textcaps_test)
+- TextVQA (textvqa)
+  - TextVQA Validation (textvqa_val)
+  - TextVQA Test (textvqa_test)
+- VizWizVQA (vizwiz_vqa)
+  - VizWizVQA Validation (vizwiz_vqa_val)
+  - VizWizVQA Test (vizwiz_vqa_test)
+- VQAv2 (vqav2)
+  - VQAv2 Validation (vqav2_val)
+  - VQAv2 Test (vqav2_test)
+- WebSRC (websrc)
+  - WebSRC Validation (websrc_val)
+  - WebSRC Test (websrc_test)
@@ -778,6 +778,7 @@ def _download_from_youtube(path):
                     force_unzip = dataset_kwargs.get("force_unzip", False)
                     cache_path = snapshot_download(repo_id=self.DATASET_PATH, repo_type="dataset", force_download=force_download, etag_timeout=60)
                     zip_files = glob(os.path.join(cache_path, "**/*.zip"), recursive=True)
+                    tar_files = glob(os.path.join(cache_path, "**/*.tar*"), recursive=True)
 
                     def unzip_video_data(zip_file):
                         import zipfile
@@ -786,10 +787,51 @@ def unzip_video_data(zip_file):
                             zip_ref.extractall(cache_dir)
                             eval_logger.info(f"Extracted all files from {zip_file} to {cache_dir}")
 
+                    def untar_video_data(tar_file):
+                        import tarfile
+
+                        with tarfile.open(tar_file, "r") as tar_ref:
+                            tar_ref.extractall(cache_dir)
+                            eval_logger.info(f"Extracted all files from {tar_file} to {cache_dir}")
+
+                    def concat_tar_parts(tar_parts, output_tar):
+                        with open(output_tar, "wb") as out_tar:
+                            from tqdm import tqdm
+
+                            for part in tqdm(sorted(tar_parts)):
+                                with open(part, "rb") as part_file:
+                                    out_tar.write(part_file.read())
+                        eval_logger.info(f"Concatenated parts {tar_parts} into {output_tar}")
+
+                    # Unzip zip files if needed
                     if force_unzip or (not os.path.exists(cache_dir) and len(zip_files) > 0):
                         for zip_file in zip_files:
                             unzip_video_data(zip_file)
 
+                    # Concatenate and extract tar files if needed
+                    if force_unzip or (not os.path.exists(cache_dir) and len(tar_files) > 0):
+                        tar_parts_dict = {}
+
+                        # Group tar parts together
+                        for tar_file in tar_files:
+                            base_name = tar_file.split(".tar")[0]
+                            if base_name not in tar_parts_dict:
+                                tar_parts_dict[base_name] = []
+                            tar_parts_dict[base_name].append(tar_file)
+
+                        # Concatenate and untar split parts
+                        for base_name, parts in tar_parts_dict.items():
+                            eval_logger.info(f"Extracting following tar files: {parts}")
+                            output_tar = base_name + ".tar"
+                            if not os.path.exists(output_tar):
+                                eval_logger.info(f"Start concatenating tar files")
+
+                                concat_tar_parts(parts, output_tar)
+                                eval_logger.info(f"Finish concatenating tar files")
+
+                            if not os.path.exists(os.path.join(cache_dir, os.path.basename(base_name))):
+                                untar_video_data(output_tar)
+
                 accelerator.wait_for_everyone()
                 dataset_kwargs.pop("cache_dir")
                 dataset_kwargs.pop("video")
 
@@ -26,7 +26,9 @@
     "reka": "Reka",
     "llava_onevision": "Llava_OneVision",
     "from_log": "FromLog",
-    "mplugOwlVideo": "mplug_Owl",
+    "mplug_owl_video": "mplug_Owl",
+    "phi3v": "Phi3v",
+    "tinyllava": "TinyLlava",
 }
 
 for model_name, model_class in AVAILABLE_MODELS.items():