feat(transformers/model): add InternVL

examples/transformers/internvl/generate.py

@@ -0,0 +1,65 @@
+import time
+
+from PIL import Image
+from transformers import GotOcr2ImageProcessor, InternVLProcessor
+
+import mindspore as ms
+
+from mindone.transformers import InternVLForConditionalGeneration
+
+MODEL_HUB = "OpenGVLab/InternVL3-1B-hf"
+image = "demo.jpeg"
+
+# Load processor
+start = time.time()
+processor = InternVLProcessor.from_pretrained(MODEL_HUB)
+# GotOcr2ImageProcessorFast does not support return_tensors="np", use GotOcr2ImageProcessor instead
+image_processor = GotOcr2ImageProcessor.from_pretrained(MODEL_HUB)
+processor.image_processor = image_processor
+print(f"Loaded InternVLProcessor in {time.time()-start:.4f}s")
+
+# Load model with bfloat16 and eager attention
+start = time.time()
+model = InternVLForConditionalGeneration.from_pretrained(
+    MODEL_HUB,
+    mindspore_dtype=ms.bfloat16,
+    attn_implementation="eager",
+)
+print(f"Loaded model in {time.time()-start:.4f}s")
+
+# load image
+image = Image.open(image)
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": image,
+            },
+            {"type": "text", "text": "Describe this image."},
+        ],
+    }
+]
+prompt = processor.apply_chat_template(messages, add_generation_prompt=True)
+
+# Tokenize + encode
+inputs = processor(text=prompt, images=[image], return_tensors="np")
+
+for k, v in inputs.items():
+    tensor = ms.Tensor(v)
+    if tensor.dtype == ms.int64:
+        tensor = tensor.astype(ms.int32)
+    else:
+        tensor = tensor.astype(model.dtype)
+    inputs[k] = tensor
+
+# Generate
+start = time.time()
+generated_ids = model.generate(**inputs, max_new_tokens=500)
+print(f"Inference in {time.time()-start:.4f}s")
+
+# Decode
+texts = processor.batch_decode(generated_ids, skip_special_tokens=True)
+print(texts)

mindone/transformers/__init__.py

@@ -693,6 +693,15 @@
         Glm4PreTrainedModel,
     )
 
+if version.parse(transformers.__version__) >= version.parse("4.52.0"):
+    from .models.internvl import (
+        InternVLForConditionalGeneration,
+        InternVLModel,
+        InternVLPreTrainedModel,
+        InternVLVisionModel,
+        InternVLVisionPreTrainedModel,
+    )
+
 if version.parse(transformers.__version__) >= version.parse("4.53.0"):
     from .models.glm4v import (
         Glm4vForConditionalGeneration,

mindone/transformers/modeling_utils.py

@@ -651,6 +651,8 @@ class PreTrainedModel(nn.Cell, ModuleUtilsMixin, GenerationMixin, PushToHubMixin
     main_input_name = "input_ids"
     model_tags = None
 
+    _checkpoint_conversion_mapping = {}
+
     _auto_class = None
     _no_split_modules = None
     _skip_keys_device_placement = None
@@ -1920,6 +1922,7 @@ def from_pretrained(
         use_flash_attention_2 = kwargs.pop("use_flash_attention_2", False)
         adapter_kwargs = kwargs.pop("adapter_kwargs", {})
         adapter_name = kwargs.pop("adapter_name", "default")
+        key_mapping = kwargs.pop("key_mapping", None)
 
         if use_auth_token is not None:
             warnings.warn(
@@ -2391,6 +2394,11 @@ def from_pretrained(
                 sharded_metadata=sharded_metadata,
                 dtype=mindspore_dtype,
                 keep_in_fp32_modules=keep_in_fp32_modules,
+                key_mapping=(
+                    key_mapping
+                    if key_mapping is not None
+                    else (getattr(cls, "_checkpoint_conversion_mapping", None) or None)
+                ),
             )
 
         if _adapter_model_path is not None:
@@ -2442,6 +2450,62 @@ def from_pretrained(
 
         return model
 
+    @staticmethod
+    def _fix_state_dict_key_on_load(key: str) -> tuple[str, bool]:
+        return key, False
+
+    def _get_key_renaming_mapping(  # NEW (HF parity)
+        self,
+        checkpoint_keys: list[str],
+        key_mapping: dict[str, str] | None = None,
+        loading_base_model_from_task_state_dict: bool = False,
+        loading_task_model_from_base_state_dict: bool = False,
+    ) -> dict[str, str]:
+        prefix = self.base_model_prefix
+        _prefix = f"{prefix}."
+
+        renamed_keys_for_log: dict[str, tuple[str, str]] = {}
+        key_renaming_mapping: dict[str, str] = {}
+
+        for key in checkpoint_keys:
+            new_key, has_changed = self._fix_state_dict_key_on_load(key)
+
+            # 2) optional regex key mapping
+            if key_mapping is not None:
+                for pattern, replacement in key_mapping.items():
+                    updated_key, n_replace = re.subn(pattern, replacement, new_key)
+                    if n_replace > 0:
+                        has_changed = True
+                        new_key = updated_key
+
+            if loading_task_model_from_base_state_dict:
+                new_key = ".".join([prefix, new_key])
+            elif loading_base_model_from_task_state_dict:
+                if not new_key.startswith(_prefix):
+                    continue
+                new_key = new_key[len(_prefix) :]
+
+            key_renaming_mapping[key] = new_key
+
+            if has_changed:
+                if key.endswith("LayerNorm.gamma"):
+                    renamed_keys_for_log["LayerNorm.gamma"] = (key, new_key)
+                elif key.endswith("LayerNorm.beta"):
+                    renamed_keys_for_log["LayerNorm.beta"] = (key, new_key)
+
+        if renamed_keys_for_log:
+            msg = (
+                f"A pretrained model of type `{self.__class__.__name__}` "
+                "contains parameters that have been renamed internally (a few are listed):\n"
+            )
+            for old_key, new_key in renamed_keys_for_log.values():
+                msg += f"* `{old_key}` -> `{new_key}`\n"
+            # optional: encourage upstream PRs as HF does
+            msg += "If you loaded from the Hub, consider submitting a PR to adjust these weights."
+            logger.info(msg)
+
+        return key_renaming_mapping
+
     @classmethod
     def _load_pretrained_model(
         cls,
@@ -2454,6 +2518,7 @@ def _load_pretrained_model(
         sharded_metadata=None,
         dtype=None,
         keep_in_fp32_modules=None,
+        key_mapping: dict[str, str] | None = None,
     ):
         model.tie_weights()
 
@@ -2470,6 +2535,17 @@ def _load_pretrained_model(
             has_prefix_module = False
             expects_prefix_module = False
 
+        loading_task_model_from_base_state_dict = (not has_prefix_module) and expects_prefix_module
+        loading_base_model_from_task_state_dict = has_prefix_module and (not expects_prefix_module)
+
+        key_renaming_mapping = model._get_key_renaming_mapping(
+            original_loaded_keys,
+            key_mapping=key_mapping,
+            loading_base_model_from_task_state_dict=loading_base_model_from_task_state_dict,
+            loading_task_model_from_base_state_dict=loading_task_model_from_base_state_dict,
+        )
+        loaded_keys = list(key_renaming_mapping.values())
+
         # Mapping loaded_keys from pt to ms
         pt2ms_mappings = _get_pt2ms_mappings(model)
         loaded_keys = _get_pt2ms_mapped_k(pt2ms_mappings, has_prefix_module, expects_prefix_module, loaded_keys, prefix)
@@ -2563,6 +2639,9 @@ def _find_mismatched_keys(
             # Whole checkpoint
             state_dict = _convert_state_dict(model, state_dict, prefix)
 
+            if key_renaming_mapping:
+                state_dict = {key_renaming_mapping[k]: v for k, v in state_dict.items() if k in key_renaming_mapping}
+
             mismatched_keys = _find_mismatched_keys(
                 state_dict,
                 model_state_dict,
@@ -2590,6 +2669,11 @@ def _find_mismatched_keys(
                 state_dict = load_state_dict(shard_file)
                 state_dict = _convert_state_dict(model, state_dict, prefix)
 
+                if key_renaming_mapping:
+                    state_dict = {
+                        key_renaming_mapping[k]: v for k, v in state_dict.items() if k in key_renaming_mapping
+                    }
+
                 # Mismatched keys contains tuples key/shape1/shape2 of weights in the checkpoint that have a shape not
                 # matching the weights in the model.
                 mismatched_keys += _find_mismatched_keys(

mindone/transformers/models/__init__.py

@@ -106,5 +106,8 @@
 if version.parse(transformers.__version__) >= version.parse("4.51.3"):
     from . import glm4
 
+if version.parse(transformers.__version__) >= version.parse("4.52.0"):
+    from . import internvl
+
 if version.parse(transformers.__version__) >= version.parse("4.53.0"):
     from . import glm4v, minimax, vjepa2

mindone/transformers/models/auto/configuration_auto.py

@@ -298,6 +298,13 @@
     CONFIG_MAPPING_NAMES.update({"glm4": "Glm4Config"})
     MODEL_NAMES_MAPPING.update({"glm4": "glm4"})
 
+if version.parse(transformers.__version__) >= version.parse("4.52.0"):
+    CONFIG_MAPPING_NAMES.update({"internvl": "InternVLConfig"})
+    CONFIG_MAPPING_NAMES.update({"internvl_vision": "InternVLVisionConfig"})
+    MODEL_NAMES_MAPPING.update({"internvl": "InternVLModel"})  # TODO: InternVL
+    MODEL_NAMES_MAPPING.update({"internvl_vision": "InternVLVisionModel"})  # TODO: InternVLVision
+    SPECIAL_MODEL_TYPE_TO_MODULE_NAME.update({"internvl_vision": "internvl"})
+
 if version.parse(transformers.__version__) >= version.parse("4.53.0"):
     CONFIG_MAPPING_NAMES.update({"minimax": "MiniMaxConfig", "vjepa2": "VJEPA2Model"})
     MODEL_NAMES_MAPPING.update({"minimax": "MiniMax", "vjepa2": "VJEPA2Model"})

mindone/transformers/models/auto/modeling_auto.py

@@ -680,6 +680,11 @@
     MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES.update({"glm4": "Glm4ForSequenceClassification"})
     MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES.update({"glm4": "Glm4ForTokenClassification"})
 
+if version.parse(transformers.__version__) >= version.parse("4.52.0"):
+    MODEL_MAPPING_NAMES.update({"internvl": "InternVLModel"})
+    MODEL_MAPPING_NAMES.update({"internvl_vision": "InternVLVisionModel"})
+    MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES.update({"internvl": "InternVLForConditionalGeneration"})
+
 if version.parse(transformers.__version__) >= version.parse("4.53.0"):
     MODEL_MAPPING_NAMES.update({"minimax": "MiniMaxModel", "vjepa2": "VJEPA2Model"})
     MODEL_FOR_CAUSAL_LM_MAPPING_NAMES.update({"minimax": "MiniMaxForCausalLM"})

mindone/transformers/models/internvl/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2025 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .modeling_internvl import (
+    InternVLForConditionalGeneration,
+    InternVLModel,
+    InternVLPreTrainedModel,
+    InternVLVisionModel,
+    InternVLVisionPreTrainedModel,
+)