Merge pull request EvolvingLMMs-Lab#125 from EvolvingLMMs-Lab/dev/interleave

[Model] aligned llava-interleave model results on video tasks

Author: Luodian

lmms_eval/__main__.py

@@ -1,12 +1,10 @@
 import os
 import yaml
 import sys
-import copy
 import json
-import logging
+
 import traceback
 import argparse
-import torch
 import numpy as np
 import datetime
 
@@ -25,10 +23,7 @@
 from lmms_eval.tasks import initialize_tasks, include_path, get_task_dict
 from lmms_eval.api.registry import ALL_TASKS
 from lmms_eval.logging_utils import WandbLogger
-from lmms_eval.utils import PathFormatter
-
-
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 def _handle_non_serializable(o):
@@ -166,9 +161,10 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
         print("└───────────────────────────────────────────────────────────────────────────────┘")
         sys.exit(1)
 
-    set_loggers(args)
-    eval_logger = logging.getLogger("lmms-eval")
-    eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
+    # reset logger
+    eval_logger.remove()
+    eval_logger.add(sys.stdout, colorize=True, level=args.verbosity)
+    eval_logger.add(sys.stderr, level=args.verbosity)
     eval_logger.info(f"Verbosity set to {args.verbosity}")
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
 
@@ -228,11 +224,6 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
 
 
 def cli_evaluate_single(args: Union[argparse.Namespace, None] = None) -> None:
-    eval_logger = logging.getLogger("lmms-eval")
-    eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
-    eval_logger.info(f"Verbosity set to {args.verbosity}")
-    os.environ["TOKENIZERS_PARALLELISM"] = "false"
-
     initialize_tasks(args.verbosity)
 
     if args.predict_only:
@@ -350,13 +341,5 @@ def print_results(args, results):
         print(evaluator.make_table(results, "groups"))
 
 
-def set_loggers(args):
-    eval_logger = logging.getLogger("lmms-eval")
-    ch = logging.StreamHandler()
-    formatter = PathFormatter("%(asctime)s [%(pathname)s:%(lineno)d] %(levelname)s %(message)s", "%m-%d %H:%M:%S", timezone=args.timezone)
-    ch.setFormatter(formatter)
-    eval_logger.addHandler(ch)
-
-
 if __name__ == "__main__":
     cli_evaluate()

lmms_eval/api/metrics.py

@@ -9,10 +9,7 @@
 import torch
 
 from lmms_eval.api.registry import register_metric, register_aggregation
-
-import logging
-
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 # Register Aggregations First

lmms_eval/api/model.py

@@ -8,9 +8,9 @@
 from lmms_eval.api.instance import Instance
 from tqdm import tqdm
 from lmms_eval import utils
-import logging
 
-eval_logger = logging.getLogger("lmms-eval")
+
+from loguru import logger as eval_logger
 
 T = TypeVar("T", bound="lmms")
 

lmms_eval/api/registry.py

@@ -1,10 +1,9 @@
 from lmms_eval.api.model import lmms
 
 from typing import Callable, Dict
-import logging
 import evaluate as hf_evaluate
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 MODEL_REGISTRY = {}
 

lmms_eval/api/task.py

@@ -2,7 +2,7 @@
 import ast
 import itertools
 import json
-import logging
+
 import os
 import random
 import re
@@ -37,7 +37,7 @@
 )
 from lmms_eval.filters import build_filter_ensemble
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 # HuggingfaceM4/NoCaps contains truncated image in test split
 # Include this inside code block to avoid error
@@ -789,17 +789,17 @@ def unzip_video_data(zip_file):
 
                     def untar_video_data(tar_file):
                         import tarfile
+
                         with tarfile.open(tar_file, "r") as tar_ref:
                             tar_ref.extractall(cache_dir)
                             eval_logger.info(f"Extracted all files from {tar_file} to {cache_dir}")
 
-
-    
                     def concat_tar_parts(tar_parts, output_tar):
-                        with open(output_tar, 'wb') as out_tar:
+                        with open(output_tar, "wb") as out_tar:
                             from tqdm import tqdm
+
                             for part in tqdm(sorted(tar_parts)):
-                                with open(part, 'rb') as part_file:
+                                with open(part, "rb") as part_file:
                                     out_tar.write(part_file.read())
                         eval_logger.info(f"Concatenated parts {tar_parts} into {output_tar}")
 
@@ -811,15 +811,14 @@ def concat_tar_parts(tar_parts, output_tar):
                     # Concatenate and extract tar files if needed
                     if force_unzip or (not os.path.exists(cache_dir) and len(tar_files) > 0):
                         tar_parts_dict = {}
-                        
+
                         # Group tar parts together
                         for tar_file in tar_files:
-                            base_name = tar_file.split('.tar')[0]
+                            base_name = tar_file.split(".tar")[0]
                             if base_name not in tar_parts_dict:
                                 tar_parts_dict[base_name] = []
                             tar_parts_dict[base_name].append(tar_file)
 
-                        
                         # Concatenate and untar split parts
                         for base_name, parts in tar_parts_dict.items():
                             eval_logger.info(f"Extracting following tar files: {parts}")

lmms_eval/evaluator.py

@@ -7,7 +7,7 @@
 from tqdm import tqdm
 
 import torch
-import logging
+
 import numpy as np
 from datasets import Image, Sequence
 
@@ -17,8 +17,6 @@
 import lmms_eval.api.metrics
 import lmms_eval.api.registry
 
-import re
-
 from lmms_eval.utils import (
     positional_deprecated,
     run_task_tests,
@@ -28,7 +26,7 @@
     simple_parse_args_string,
 )
 
-eval_logger = logging.getLogger("lmms-eval")
+from loguru import logger as eval_logger
 
 
 @positional_deprecated

lmms_eval/logging_utils.py

@@ -1,6 +1,6 @@
 # Code mostly from: https://github.com/EleutherAI/lm-evaluation-harness/pull/1339, credit to: https://github.com/ayulockin
 import copy
-import logging
+
 import re
 import os
 import json
@@ -9,13 +9,10 @@
 import numpy as np
 from datetime import datetime
 from typing import Any, Dict, List, Literal, Tuple, Union
-
 from packaging.version import Version
-
 from lmms_eval import utils
 import tenacity
-
-logger = logging.getLogger(__name__)
+from loguru import logger
 
 try:
     import wandb

lmms_eval/models/__init__.py

@@ -1,7 +1,8 @@
-import os
-import hf_transfer
+from loguru import logger
+import sys
 
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+logger.remove()
+logger.add(sys.stdout, level="WARNING")
 
 AVAILABLE_MODELS = {
     "llava": "Llava",
@@ -22,18 +23,18 @@
     "idefics2": "Idefics2",
     "internvl": "InternVLChat",
     "gemini_api": "GeminiAPI",
-    "gemini_model": "GeminiModel",
     "reka": "Reka",
-    "llava_onevision": "Llava_OneVision",
     "from_log": "FromLog",
     "mplug_owl_video": "mplug_Owl",
     "phi3v": "Phi3v",
     "tinyllava": "TinyLlava",
-    "llava_hf" : "LlavaHf",
+    "llava_hf": "LlavaHf",
+    "longva": "LongVA",
 }
 
 for model_name, model_class in AVAILABLE_MODELS.items():
     try:
         exec(f"from .{model_name} import {model_class}")
-    except ImportError:
+    except ImportError as e:
+        # logger.warning(f"Failed to import {model_class} from {model_name}: {e}")
         pass

lmms_eval/models/batch_gpt4.py

@@ -2,7 +2,7 @@
 from copy import deepcopy
 from io import BytesIO
 import base64
-import logging
+
 import os
 import time
 import json
@@ -20,14 +20,13 @@
 from lmms_eval.api.instance import Instance
 from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
-from lmms_eval import utils
+from loguru import logger as eval_logger
 
 # Conditional imports
 try:
     from decord import VideoReader, cpu
 except ImportError:
-    eval_logger = logging.getLogger("lmms-eval")
-    eval_logger.info("Decord is not installed. Video input will not be supported.")
+    eval_logger.warning("Decord is not installed. Video input will not be supported.")
 
 # Constants and global configurations
 API_TYPE = os.getenv("API_TYPE", "openai")

lmms_eval/models/claude.py

@@ -5,28 +5,28 @@
 import json
 from typing import List, Tuple, Union
 from tqdm import tqdm
-import requests as url_requests
 import time
-import logging
 
 from lmms_eval.api.instance import Instance
 from lmms_eval.api.model import lmms
 from lmms_eval.api.registry import register_model
-from lmms_eval import utils
 
 from accelerate import Accelerator, DistributedType
 
 from PIL import Image
 
 NUM_SECONDS_TO_SLEEP = 5
-eval_logger = logging.getLogger("lmms-eval")
+
+from loguru import logger
+
+eval_logger = logger
 
 try:
     import anthropic
     from decord import VideoReader, cpu
     import numpy as np
 except Exception as e:
-    eval_logger.error(f"Error importing claude: {e}")
+    eval_logger.warning(f"Error importing claude: {e}")
 
 API_URL = os.getenv("ANTHROPIC_API_URL", "https://api.anthropic.com/v1/complete")
 API_KEY = os.getenv("ANTHROPIC_API_KEY", "YOUR_API_KEY")