marlin fp32 mode should also be enabled if kernel was selected due to backend.auto (#1318)

Signed-off-by: Qubitium <[email protected]>

Author: Qubitium

gptqmodel/nn_modules/qlinear/marlin.py

@@ -27,6 +27,7 @@
 from torch.nn.parameter import Parameter
 
 from ...models._const import DEVICE, PLATFORM
+from ...utils.logger import setup_logger
 from ...utils.rocm import IS_ROCM
 
 marlin_import_exception = None
@@ -35,6 +36,8 @@
 except ImportError as e:
     marlin_import_exception = e
 
+logger = setup_logger()
+fp32_warning_logged = False
 
 GPTQ_MARLIN_TILE = 16
 GPTQ_MARLIN_MIN_THREAD_N = 64
@@ -163,7 +166,6 @@ def apply_gptq_marlin_linear(
 
     return output.reshape(out_shape)
 
-
 class MarlinQuantLinear(BaseQuantLinear):
     SUPPORTS_BITS = [4, 8]
     SUPPORTS_GROUP_SIZE = [-1, 32, 64, 128]
@@ -220,7 +222,12 @@ def __init__(
             **kwargs)
 
         # toggle fp32 mode depending on MARLIN or MARLIN_FP16 backend
-        self.fp32 = True if self.backend is BACKEND.MARLIN else False
+        self.fp32 = True if self.backend in [BACKEND.MARLIN, BACKEND.AUTO] else False
+
+        global fp32_warning_logged
+        if not fp32_warning_logged:
+            fp32_warning_logged = True
+            logger.warn("Kernel: Marlin FP16 mode is activated with reduced accuracy. Use default Marlin model for improved inference quality.")
 
         # Determine sharding
         if marlin_repeat_scales_on_all_ranks(desc_act,

gptqmodel/utils/logger.py

@@ -84,6 +84,10 @@ def _process(self, op: Callable, msg, *args, **kwargs):
     handler.flush = sys.stdout.flush
     logger.addHandler(handler)
 
+    # fix warnings about warn() deprecated
+    if hasattr(logger, "warning"):
+        logger.warn = logger.warning
+
     return logger