From d172316559781063520f5258f4edea943f050f24 Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 26 Aug 2026 09:19:25 +0000 Subject: [PATCH 1/8] part1 --- src/transformers/models/afmoe/modeling_afmoe.py | 1 + src/transformers/models/aimv2/modeling_aimv2.py | 1 + src/transformers/models/aria/modeling_aria.py | 1 + src/transformers/models/axk1/modeling_axk1.py | 1 + src/transformers/models/axk2/modeling_axk2.py | 1 + src/transformers/models/bamba/modeling_bamba.py | 1 + src/transformers/models/bitnet/modular_bitnet.py | 2 ++ src/transformers/models/cohere/modeling_cohere.py | 2 ++ src/transformers/models/cohere2/modeling_cohere2.py | 2 ++ src/transformers/models/cohere2_moe/modeling_cohere2_moe.py | 1 + .../models/cohere_compass/modeling_cohere_compass.py | 1 + src/transformers/models/csm/modeling_csm.py | 1 + src/transformers/models/cwm/modeling_cwm.py | 1 + .../models/deepseek_ocr2/modeling_deepseek_ocr2.py | 1 + src/transformers/models/deepseek_ocr2/modular_deepseek_ocr2.py | 3 ++- src/transformers/models/deepseek_v2/modeling_deepseek_v2.py | 1 + src/transformers/models/deepseek_v3/modeling_deepseek_v3.py | 1 + src/transformers/models/deepseek_v32/modeling_deepseek_v32.py | 1 + src/transformers/models/deepseek_v4/modeling_deepseek_v4.py | 1 + src/transformers/models/deimv2/modeling_deimv2.py | 1 + src/transformers/models/diffllama/modeling_diffllama.py | 1 + src/transformers/models/dinov3_vit/modular_dinov3_vit.py | 3 ++- src/transformers/models/doge/modeling_doge.py | 1 + src/transformers/models/dots1/modeling_dots1.py | 1 + src/transformers/models/emu3/modeling_emu3.py | 1 + src/transformers/models/ernie4_5/modeling_ernie4_5.py | 1 + src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py | 1 + .../models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py | 1 + src/transformers/models/esmc/modeling_esmc.py | 1 + src/transformers/models/eurobert/modeling_eurobert.py | 1 + src/transformers/models/evolla/modeling_evolla.py | 1 + src/transformers/models/exaone4/modeling_exaone4.py | 1 + src/transformers/models/exaone_moe/modeling_exaone_moe.py | 1 + src/transformers/models/falcon_h1/modeling_falcon_h1.py | 1 + src/transformers/models/flex_olmo/modeling_flex_olmo.py | 1 + src/transformers/models/llama/modeling_llama.py | 1 + 36 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/afmoe/modeling_afmoe.py b/src/transformers/models/afmoe/modeling_afmoe.py index bf79f260355b..69595083a045 100644 --- a/src/transformers/models/afmoe/modeling_afmoe.py +++ b/src/transformers/models/afmoe/modeling_afmoe.py @@ -119,6 +119,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class AfmoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index ec1e30d493f7..ae6854c3a730 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -97,6 +97,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Aimv2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 59f307883293..0d83fe4991bd 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -215,6 +215,7 @@ def forward(self, key_value_states: torch.Tensor, attn_mask: torch.Tensor | None return out +@use_kernel_forward_from_hub("SwiGLUMLP") class AriaSharedExpertsMLP(nn.Module): """ Shared Expert MLP for shared experts. diff --git a/src/transformers/models/axk1/modeling_axk1.py b/src/transformers/models/axk1/modeling_axk1.py index f273e76374b5..d6db63541715 100644 --- a/src/transformers/models/axk1/modeling_axk1.py +++ b/src/transformers/models/axk1/modeling_axk1.py @@ -126,6 +126,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class AXK1MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/axk2/modeling_axk2.py b/src/transformers/models/axk2/modeling_axk2.py index 25506bf85876..b363fe752131 100644 --- a/src/transformers/models/axk2/modeling_axk2.py +++ b/src/transformers/models/axk2/modeling_axk2.py @@ -381,6 +381,7 @@ def forward( return final_hidden_states +@use_kernel_forward_from_hub("SwiGLUMLP") class AXK2MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/bamba/modeling_bamba.py b/src/transformers/models/bamba/modeling_bamba.py index 2a06e67cd53e..46e8959020f9 100644 --- a/src/transformers/models/bamba/modeling_bamba.py +++ b/src/transformers/models/bamba/modeling_bamba.py @@ -795,6 +795,7 @@ def forward( return contextualized_states +@use_kernel_forward_from_hub("SwiGLUMLP") class BambaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/bitnet/modular_bitnet.py b/src/transformers/models/bitnet/modular_bitnet.py index 7f3a248766b2..dfaa5a16227f 100644 --- a/src/transformers/models/bitnet/modular_bitnet.py +++ b/src/transformers/models/bitnet/modular_bitnet.py @@ -22,6 +22,7 @@ from ...modeling_utils import ALL_ATTENTION_FUNCTIONS from ...processing_utils import Unpack from ...utils import logging +from ...utils.generic import no_inherit_decorator from ..gemma.modeling_gemma import GemmaMLP from ..llama.modeling_llama import ( LlamaAttention, @@ -42,6 +43,7 @@ class BitNetRMSNorm(LlamaRMSNorm): pass +@no_inherit_decorator class BitNetMLP(GemmaMLP): def __init__(self, config: BitNetConfig): super().__init__(config) diff --git a/src/transformers/models/cohere/modeling_cohere.py b/src/transformers/models/cohere/modeling_cohere.py index adcbc0d26d97..4b61d88df90b 100644 --- a/src/transformers/models/cohere/modeling_cohere.py +++ b/src/transformers/models/cohere/modeling_cohere.py @@ -34,6 +34,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer @@ -119,6 +120,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class CohereMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/cohere2/modeling_cohere2.py b/src/transformers/models/cohere2/modeling_cohere2.py index cb920ec8b236..9c3077bc9caa 100644 --- a/src/transformers/models/cohere2/modeling_cohere2.py +++ b/src/transformers/models/cohere2/modeling_cohere2.py @@ -26,6 +26,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_layers import GradientCheckpointingLayer from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast @@ -253,6 +254,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Cohere2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py b/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py index 3b6469405e6f..b06d1deed09e 100644 --- a/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py +++ b/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py @@ -82,6 +82,7 @@ def forward(self, hidden_states): return hidden_states.to(input_dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class Cohere2MoeMLP(nn.Module): def __init__(self, config: Cohere2MoeConfig, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/cohere_compass/modeling_cohere_compass.py b/src/transformers/models/cohere_compass/modeling_cohere_compass.py index b9031b136477..70ebafae417d 100644 --- a/src/transformers/models/cohere_compass/modeling_cohere_compass.py +++ b/src/transformers/models/cohere_compass/modeling_cohere_compass.py @@ -161,6 +161,7 @@ def apply_interleaved_mrope(self, freqs, mrope_section): return freqs_t +@use_kernel_forward_from_hub("SwiGLUMLP") class CohereCompassMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/csm/modeling_csm.py b/src/transformers/models/csm/modeling_csm.py index 3b61827a7c0b..ff61e41f1695 100644 --- a/src/transformers/models/csm/modeling_csm.py +++ b/src/transformers/models/csm/modeling_csm.py @@ -174,6 +174,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class CsmMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/cwm/modeling_cwm.py b/src/transformers/models/cwm/modeling_cwm.py index 546d9a14c6ce..c9751da9cebe 100644 --- a/src/transformers/models/cwm/modeling_cwm.py +++ b/src/transformers/models/cwm/modeling_cwm.py @@ -249,6 +249,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class CwmMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py b/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py index 90676efdd97c..ad320db831cf 100644 --- a/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py +++ b/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py @@ -1118,6 +1118,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class DeepseekOcr2TextMLP(nn.Module): def __init__(self, config: DeepseekOcr2TextConfig, hidden_size=None, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_ocr2/modular_deepseek_ocr2.py b/src/transformers/models/deepseek_ocr2/modular_deepseek_ocr2.py index 989034e8431b..6b2e952a43f2 100644 --- a/src/transformers/models/deepseek_ocr2/modular_deepseek_ocr2.py +++ b/src/transformers/models/deepseek_ocr2/modular_deepseek_ocr2.py @@ -40,7 +40,7 @@ from ...modeling_utils import PreTrainedModel from ...processing_utils import Unpack from ...utils import TensorType, TransformersKwargs, auto_docstring, can_return_tuple, logging -from ...utils.generic import merge_with_config_defaults +from ...utils.generic import merge_with_config_defaults, no_inherit_decorator from ...utils.import_utils import requires from ...utils.output_capturing import capture_outputs from ..deepseek_v2.configuration_deepseek_v2 import DeepseekV2Config @@ -783,6 +783,7 @@ def forward(self, pixel_values: torch.FloatTensor, **kwargs) -> BaseModelOutput: return BaseModelOutput(last_hidden_state=hidden_states) +@no_inherit_decorator class DeepseekOcr2VisionMLP(Qwen2MLP): pass diff --git a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py index 0968526b2485..5e6ca6dea88e 100644 --- a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py +++ b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py @@ -137,6 +137,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states +@use_kernel_forward_from_hub("SwiGLUMLP") class DeepseekV2MLP(nn.Module): def __init__(self, config: DeepseekV2Config, hidden_size=None, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index 79f9149f2e26..8d57b1199289 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -112,6 +112,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class DeepseekV3MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py b/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py index fd407bf47cbd..647c24019760 100644 --- a/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py +++ b/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py @@ -493,6 +493,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class DeepseekV32MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py b/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py index a399e426feea..745b3ea2bc6e 100644 --- a/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py +++ b/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py @@ -962,6 +962,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return (pre.unsqueeze(-1) * x).sum(dim=2).to(x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class DeepseekV4MLP(nn.Module): def __init__(self, config: DeepseekV4Config): super().__init__() diff --git a/src/transformers/models/deimv2/modeling_deimv2.py b/src/transformers/models/deimv2/modeling_deimv2.py index 44905ad1ed37..d881b94719da 100644 --- a/src/transformers/models/deimv2/modeling_deimv2.py +++ b/src/transformers/models/deimv2/modeling_deimv2.py @@ -177,6 +177,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Deimv2SwiGLUFFN(nn.Module): def __init__(self, config: Deimv2Config): super().__init__() diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index 2f2f85b62052..54356c80dc54 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -49,6 +49,7 @@ from .configuration_diffllama import DiffLlamaConfig +@use_kernel_forward_from_hub("SwiGLUMLP") class DiffLlamaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/dinov3_vit/modular_dinov3_vit.py b/src/transformers/models/dinov3_vit/modular_dinov3_vit.py index 1af9b58d2d2e..73df746eb4af 100644 --- a/src/transformers/models/dinov3_vit/modular_dinov3_vit.py +++ b/src/transformers/models/dinov3_vit/modular_dinov3_vit.py @@ -38,7 +38,7 @@ from ...processing_utils import Unpack from ...pytorch_utils import compile_compatible_method_lru_cache from ...utils import TransformersKwargs, auto_docstring, logging -from ...utils.generic import can_return_tuple, maybe_autocast, merge_with_config_defaults +from ...utils.generic import can_return_tuple, maybe_autocast, merge_with_config_defaults, no_inherit_decorator from ...utils.output_capturing import capture_outputs from ..swin.modeling_swin import SwinDropPath from .configuration_dinov3_vit import DINOv3ViTConfig @@ -300,6 +300,7 @@ class DINOv3ViTMLP(ArceeMLP): pass +@no_inherit_decorator class DINOv3ViTGatedMLP(LlamaMLP): pass diff --git a/src/transformers/models/doge/modeling_doge.py b/src/transformers/models/doge/modeling_doge.py index 6fd46ac81f9f..3cfe429eb79c 100644 --- a/src/transformers/models/doge/modeling_doge.py +++ b/src/transformers/models/doge/modeling_doge.py @@ -371,6 +371,7 @@ def prepare_dynamic_mask( return attn_mask +@use_kernel_forward_from_hub("SwiGLUMLP") class DogeMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/dots1/modeling_dots1.py b/src/transformers/models/dots1/modeling_dots1.py index 04fc35253bbc..502f5d2fbad5 100644 --- a/src/transformers/models/dots1/modeling_dots1.py +++ b/src/transformers/models/dots1/modeling_dots1.py @@ -260,6 +260,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Dots1MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index f3fd4ce0f9f0..a208ba21467f 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -216,6 +216,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Emu3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/ernie4_5/modeling_ernie4_5.py b/src/transformers/models/ernie4_5/modeling_ernie4_5.py index 86798b16a5c8..0b0889be2f63 100644 --- a/src/transformers/models/ernie4_5/modeling_ernie4_5.py +++ b/src/transformers/models/ernie4_5/modeling_ernie4_5.py @@ -95,6 +95,7 @@ def forward(self, x, position_ids): return cos, sin +@use_kernel_forward_from_hub("SwiGLUMLP") class Ernie4_5MLP(nn.Module): def __init__(self, config: Ernie4_5Config): super().__init__() diff --git a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py index a347f9a7358f..9addf5c08c3a 100644 --- a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py +++ b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py @@ -63,6 +63,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Ernie4_5_MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index f07ea98b14ad..ab911c8d1c45 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -291,6 +291,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Ernie4_5_VLMoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/esmc/modeling_esmc.py b/src/transformers/models/esmc/modeling_esmc.py index f3150aacb2c4..dbe8d7a94d13 100644 --- a/src/transformers/models/esmc/modeling_esmc.py +++ b/src/transformers/models/esmc/modeling_esmc.py @@ -110,6 +110,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class EsmcMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/eurobert/modeling_eurobert.py b/src/transformers/models/eurobert/modeling_eurobert.py index 931c9e4366ef..1e4f96051278 100644 --- a/src/transformers/models/eurobert/modeling_eurobert.py +++ b/src/transformers/models/eurobert/modeling_eurobert.py @@ -200,6 +200,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class EuroBertMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/evolla/modeling_evolla.py b/src/transformers/models/evolla/modeling_evolla.py index 45ad92b89f57..67151b1abcf4 100644 --- a/src/transformers/models/evolla/modeling_evolla.py +++ b/src/transformers/models/evolla/modeling_evolla.py @@ -1083,6 +1083,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class EvollaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/exaone4/modeling_exaone4.py b/src/transformers/models/exaone4/modeling_exaone4.py index 6a35803a94b2..15944ecfbba7 100644 --- a/src/transformers/models/exaone4/modeling_exaone4.py +++ b/src/transformers/models/exaone4/modeling_exaone4.py @@ -265,6 +265,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Exaone4MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/exaone_moe/modeling_exaone_moe.py b/src/transformers/models/exaone_moe/modeling_exaone_moe.py index 5033eae0cdd3..8567144fe242 100644 --- a/src/transformers/models/exaone_moe/modeling_exaone_moe.py +++ b/src/transformers/models/exaone_moe/modeling_exaone_moe.py @@ -208,6 +208,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class ExaoneMoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/falcon_h1/modeling_falcon_h1.py b/src/transformers/models/falcon_h1/modeling_falcon_h1.py index 9edbd2b59b32..468e592e4838 100644 --- a/src/transformers/models/falcon_h1/modeling_falcon_h1.py +++ b/src/transformers/models/falcon_h1/modeling_falcon_h1.py @@ -822,6 +822,7 @@ def forward( return contextualized_states +@use_kernel_forward_from_hub("SwiGLUMLP") class FalconH1MLP(nn.Module): def __init__(self, config: FalconH1Config): super().__init__() diff --git a/src/transformers/models/flex_olmo/modeling_flex_olmo.py b/src/transformers/models/flex_olmo/modeling_flex_olmo.py index 2654532668b9..831c34aa9cec 100644 --- a/src/transformers/models/flex_olmo/modeling_flex_olmo.py +++ b/src/transformers/models/flex_olmo/modeling_flex_olmo.py @@ -117,6 +117,7 @@ def forward(self, x, position_ids): return cos, sin +@use_kernel_forward_from_hub("SwiGLUMLP") class FlexOlmoMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index ceaea60027fa..13b7c1370046 100644 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -160,6 +160,7 @@ def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): return q_embed, k_embed +@use_kernel_forward_from_hub("SwiGLUMLP") class LlamaMLP(nn.Module): def __init__(self, config): super().__init__() From 954ebcde8dd928f7e402a322c3dfb1d5f4c8a509 Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 26 Aug 2026 10:31:58 +0000 Subject: [PATCH 2/8] p2 --- .../models/diffusion_gemma/modeling_diffusion_gemma.py | 3 ++- src/transformers/models/gemma/modeling_gemma.py | 1 + src/transformers/models/gemma/modular_gemma.py | 2 ++ src/transformers/models/gemma2/modeling_gemma2.py | 1 + src/transformers/models/gemma3/modeling_gemma3.py | 1 + src/transformers/models/gemma3n/modular_gemma3n.py | 3 ++- src/transformers/models/gemma4/modeling_gemma4.py | 4 +++- .../models/gemma4_unified/modeling_gemma4_unified.py | 2 ++ src/transformers/models/glm4_moe/modeling_glm4_moe.py | 1 + .../models/glm4_moe_lite/modeling_glm4_moe_lite.py | 1 + src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 1 + src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py | 1 + src/transformers/models/granite/modeling_granite.py | 1 + .../models/granite4_vision/modeling_granite4_vision.py | 1 + src/transformers/models/granite_swa/modeling_granite_swa.py | 1 + src/transformers/models/helium/modeling_helium.py | 2 ++ .../models/higgs_audio_v2/modeling_higgs_audio_v2.py | 1 + src/transformers/models/hrm_text/modeling_hrm_text.py | 1 + .../models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py | 1 + .../models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py | 1 + src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py | 1 + src/transformers/models/hy_v3/modeling_hy_v3.py | 1 + src/transformers/models/hyperclovax/modeling_hyperclovax.py | 1 + src/transformers/models/inkling/modeling_inkling.py | 1 + src/transformers/models/jamba/modeling_jamba.py | 1 + src/transformers/models/laguna/modeling_laguna.py | 1 + .../models/longcat_flash/modeling_longcat_flash.py | 1 + 27 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py b/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py index dedcf39a4c05..8580fd6a985d 100644 --- a/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py +++ b/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py @@ -28,7 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...configuration_utils import PreTrainedConfig -from ...integrations import use_experts_implementation +from ...integrations import use_experts_implementation, use_kernel_forward_from_hub from ...masking_utils import ( ALL_MASK_ATTENTION_FUNCTIONS, bidirectional_mask_function, @@ -497,6 +497,7 @@ def append_to_cache( return keys, values +@use_kernel_forward_from_hub("GeGLUMLP") class DiffusionGemmaText4MLP(nn.Module): def __init__(self, config: DiffusionGemmaTextConfig, layer_idx: int): super().__init__() diff --git a/src/transformers/models/gemma/modeling_gemma.py b/src/transformers/models/gemma/modeling_gemma.py index 91bbfcd58476..1b40b3ce1e96 100644 --- a/src/transformers/models/gemma/modeling_gemma.py +++ b/src/transformers/models/gemma/modeling_gemma.py @@ -81,6 +81,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" +@use_kernel_forward_from_hub("GeGLUMLP") class GemmaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/gemma/modular_gemma.py b/src/transformers/models/gemma/modular_gemma.py index b47fca63a2d1..b6c6b957aa15 100644 --- a/src/transformers/models/gemma/modular_gemma.py +++ b/src/transformers/models/gemma/modular_gemma.py @@ -21,6 +21,7 @@ from ... import initialization as init from ...cache_utils import Cache, DynamicCache from ...configuration_utils import PreTrainedConfig +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask from ...modeling_outputs import BaseModelOutputWithPast from ...modeling_rope_utils import RopeParameters @@ -136,6 +137,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" +@use_kernel_forward_from_hub("GeGLUMLP") class GemmaMLP(LlamaMLP): def __init__(self, config): super().__init__(config) diff --git a/src/transformers/models/gemma2/modeling_gemma2.py b/src/transformers/models/gemma2/modeling_gemma2.py index 253c8a44758f..fd586430ca5e 100644 --- a/src/transformers/models/gemma2/modeling_gemma2.py +++ b/src/transformers/models/gemma2/modeling_gemma2.py @@ -66,6 +66,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" +@use_kernel_forward_from_hub("GeGLUMLP") class Gemma2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index e870ec1ad81a..4649263243f2 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -117,6 +117,7 @@ def forward(self, input_ids: torch.Tensor): return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype) +@use_kernel_forward_from_hub("GeGLUMLP") class Gemma3MLP(nn.Module): def __init__(self, config: Gemma3TextConfig): super().__init__() diff --git a/src/transformers/models/gemma3n/modular_gemma3n.py b/src/transformers/models/gemma3n/modular_gemma3n.py index 3e4f2331d148..9132eece54d8 100644 --- a/src/transformers/models/gemma3n/modular_gemma3n.py +++ b/src/transformers/models/gemma3n/modular_gemma3n.py @@ -40,7 +40,7 @@ logging, torch_compilable_check, ) -from ...utils.generic import merge_with_config_defaults +from ...utils.generic import merge_with_config_defaults, no_inherit_decorator from ...utils.output_capturing import capture_outputs from ..auto import AutoModel from ..gemma2.modeling_gemma2 import ( @@ -1303,6 +1303,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states + normed_laurel_hidden_states +@no_inherit_decorator class Gemma3nTextMLP(Gemma2MLP): def __init__(self, config: Gemma3nTextConfig, layer_idx: int = 0): super().__init__(config) diff --git a/src/transformers/models/gemma4/modeling_gemma4.py b/src/transformers/models/gemma4/modeling_gemma4.py index 79f626e86996..59de0df4491e 100644 --- a/src/transformers/models/gemma4/modeling_gemma4.py +++ b/src/transformers/models/gemma4/modeling_gemma4.py @@ -33,7 +33,7 @@ from ...cache_utils import Cache, DynamicCache from ...configuration_utils import PreTrainedConfig from ...generation import GenerationMixin -from ...integrations import use_experts_implementation +from ...integrations import use_experts_implementation, use_kernel_forward_from_hub from ...masking_utils import ( _preprocess_mask_arguments, blockwise_overlay, @@ -688,6 +688,7 @@ def forward( return hidden_states, padding_positions +@use_kernel_forward_from_hub("GeGLUMLP") class Gemma4VisionMLP(nn.Module): def __init__(self, config: Gemma4VisionConfig): super().__init__() @@ -1062,6 +1063,7 @@ def forward( return BaseModelOutputWithPast(last_hidden_state=hidden_states) +@use_kernel_forward_from_hub("GeGLUMLP") class Gemma4TextMLP(nn.Module): def __init__(self, config: Gemma4TextConfig, layer_idx: int): super().__init__() diff --git a/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py b/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py index 9a897479a360..05836e7ea2b2 100644 --- a/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py +++ b/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py @@ -29,6 +29,7 @@ from ...cache_utils import Cache, DynamicCache from ...configuration_utils import PreTrainedConfig from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import ( _preprocess_mask_arguments, blockwise_overlay, @@ -455,6 +456,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("GeGLUMLP") class Gemma4UnifiedTextMLP(nn.Module): def __init__(self, config: Gemma4UnifiedTextConfig, layer_idx: int): super().__init__() diff --git a/src/transformers/models/glm4_moe/modeling_glm4_moe.py b/src/transformers/models/glm4_moe/modeling_glm4_moe.py index 539da895bd6d..29e7b729f793 100644 --- a/src/transformers/models/glm4_moe/modeling_glm4_moe.py +++ b/src/transformers/models/glm4_moe/modeling_glm4_moe.py @@ -260,6 +260,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Glm4MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py index 490317855d5d..21607f4be044 100644 --- a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py +++ b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py @@ -357,6 +357,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Glm4MoeLiteMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index fef430da0cbe..8275cb7cb96e 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -315,6 +315,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states +@use_kernel_forward_from_hub("SwiGLUMLP") class Glm4vMoeTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py index 2c3f334dff0e..2b143b24a423 100644 --- a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py +++ b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py @@ -470,6 +470,7 @@ def forward( return attn_output, attn_weights, topk_indices +@use_kernel_forward_from_hub("SwiGLUMLP") class GlmMoeDsaMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/granite/modeling_granite.py b/src/transformers/models/granite/modeling_granite.py index ba4affd2c93b..b1ebba4b1675 100644 --- a/src/transformers/models/granite/modeling_granite.py +++ b/src/transformers/models/granite/modeling_granite.py @@ -200,6 +200,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class GraniteMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/granite4_vision/modeling_granite4_vision.py b/src/transformers/models/granite4_vision/modeling_granite4_vision.py index f84d6593237c..89df07bafbf1 100644 --- a/src/transformers/models/granite4_vision/modeling_granite4_vision.py +++ b/src/transformers/models/granite4_vision/modeling_granite4_vision.py @@ -428,6 +428,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Granite4VisionTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/granite_swa/modeling_granite_swa.py b/src/transformers/models/granite_swa/modeling_granite_swa.py index 3ff773877342..c81eaa39366a 100644 --- a/src/transformers/models/granite_swa/modeling_granite_swa.py +++ b/src/transformers/models/granite_swa/modeling_granite_swa.py @@ -217,6 +217,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class GraniteSWAMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/helium/modeling_helium.py b/src/transformers/models/helium/modeling_helium.py index 1403cd7def6f..a46cf7b926d9 100644 --- a/src/transformers/models/helium/modeling_helium.py +++ b/src/transformers/models/helium/modeling_helium.py @@ -27,6 +27,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask from ...modeling_layers import ( GenericForSequenceClassification, @@ -118,6 +119,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class HeliumMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py b/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py index 12a21f886a9f..04b717ded41c 100644 --- a/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py +++ b/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py @@ -45,6 +45,7 @@ logger = logging.get_logger(__name__) +@use_kernel_forward_from_hub("SwiGLUMLP") class HiggsAudioV2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/hrm_text/modeling_hrm_text.py b/src/transformers/models/hrm_text/modeling_hrm_text.py index 315e997c4fa8..d9b145215509 100644 --- a/src/transformers/models/hrm_text/modeling_hrm_text.py +++ b/src/transformers/models/hrm_text/modeling_hrm_text.py @@ -67,6 +67,7 @@ def extra_repr(self): return f"eps={self.eps}" +@use_kernel_forward_from_hub("SwiGLUMLP") class HrmTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py index 7dccbbb44311..9777ff9a5de2 100644 --- a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py +++ b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py @@ -64,6 +64,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class HunYuanDenseV1MLP(nn.Module): def __init__(self, config: HunYuanDenseV1Config, layer_idx=None, is_shared_mlp=False): super().__init__() diff --git a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py index 60c69c1999f3..80b4c2e6bf51 100644 --- a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py +++ b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py @@ -63,6 +63,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class HunYuanMoEV1MLP(nn.Module): def __init__(self, config: HunYuanMoEV1Config): super().__init__() diff --git a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py index 116f36c138b2..17f9a84a950d 100644 --- a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py +++ b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py @@ -577,6 +577,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class HunYuanVLMLP(nn.Module): def __init__(self, config: HunYuanVLConfig, layer_idx=None, is_shared_mlp=False): super().__init__() diff --git a/src/transformers/models/hy_v3/modeling_hy_v3.py b/src/transformers/models/hy_v3/modeling_hy_v3.py index 5f9958d0fcb0..476fb0a40ee0 100644 --- a/src/transformers/models/hy_v3/modeling_hy_v3.py +++ b/src/transformers/models/hy_v3/modeling_hy_v3.py @@ -120,6 +120,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class HYV3MLP(nn.Module): def __init__(self, config: HYV3Config, intermediate_size: int | None = None): super().__init__() diff --git a/src/transformers/models/hyperclovax/modeling_hyperclovax.py b/src/transformers/models/hyperclovax/modeling_hyperclovax.py index b869e984b315..6401746782e4 100644 --- a/src/transformers/models/hyperclovax/modeling_hyperclovax.py +++ b/src/transformers/models/hyperclovax/modeling_hyperclovax.py @@ -258,6 +258,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class HyperCLOVAXMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/inkling/modeling_inkling.py b/src/transformers/models/inkling/modeling_inkling.py index 0bb061b3ea37..017ec2683c59 100644 --- a/src/transformers/models/inkling/modeling_inkling.py +++ b/src/transformers/models/inkling/modeling_inkling.py @@ -282,6 +282,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("GeGLUMLP") class InklingMLP(nn.Module): def __init__(self, config: InklingTextConfig): super().__init__() diff --git a/src/transformers/models/jamba/modeling_jamba.py b/src/transformers/models/jamba/modeling_jamba.py index 70fc8e44b31f..bfd9f76708ec 100755 --- a/src/transformers/models/jamba/modeling_jamba.py +++ b/src/transformers/models/jamba/modeling_jamba.py @@ -611,6 +611,7 @@ def forward( return contextualized_states +@use_kernel_forward_from_hub("SwiGLUMLP") class JambaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/laguna/modeling_laguna.py b/src/transformers/models/laguna/modeling_laguna.py index edaf222a815a..1eae703c255c 100644 --- a/src/transformers/models/laguna/modeling_laguna.py +++ b/src/transformers/models/laguna/modeling_laguna.py @@ -138,6 +138,7 @@ def forward(self, x, position_ids, layer_type): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class LagunaMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/longcat_flash/modeling_longcat_flash.py b/src/transformers/models/longcat_flash/modeling_longcat_flash.py index 345133f3e398..3694d3f1a5b9 100644 --- a/src/transformers/models/longcat_flash/modeling_longcat_flash.py +++ b/src/transformers/models/longcat_flash/modeling_longcat_flash.py @@ -124,6 +124,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class LongcatFlashMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() From 5e63349f97a435a24f59431cf07017bf2ec27e37 Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 26 Aug 2026 10:56:48 +0000 Subject: [PATCH 3/8] p3 --- src/transformers/models/mellum/modeling_mellum.py | 1 + src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py | 1 + src/transformers/models/minicpm3/modeling_minicpm3.py | 1 + src/transformers/models/ministral/modeling_ministral.py | 1 + src/transformers/models/ministral3/modeling_ministral3.py | 1 + src/transformers/models/mistral/modeling_mistral.py | 1 + src/transformers/models/mistral4/modeling_mistral4.py | 1 + .../models/moonshine_streaming/modeling_moonshine_streaming.py | 2 ++ src/transformers/models/muse_glimmer/modeling_muse_glimmer.py | 1 + .../muse_glimmer_assistant/modeling_muse_glimmer_assistant.py | 1 + src/transformers/models/nomic_bert/modeling_nomic_bert.py | 1 + src/transformers/models/nomic_bert/modular_nomic_bert.py | 2 ++ src/transformers/models/olmo/modeling_olmo.py | 1 + src/transformers/models/olmo2/modeling_olmo2.py | 1 + src/transformers/models/olmo3/modeling_olmo3.py | 1 + src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py | 1 + src/transformers/models/olmoe/modeling_olmoe.py | 1 + src/transformers/models/olmoe/modular_olmoe.py | 2 ++ src/transformers/models/ovis2/modeling_ovis2.py | 2 ++ src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py | 1 + src/transformers/models/pe_audio/modeling_pe_audio.py | 1 + .../models/pe_audio_video/modeling_pe_audio_video.py | 1 + src/transformers/models/pe_video/modeling_pe_video.py | 1 + 23 files changed, 27 insertions(+) diff --git a/src/transformers/models/mellum/modeling_mellum.py b/src/transformers/models/mellum/modeling_mellum.py index c4be60ecf723..6f92dff8496a 100644 --- a/src/transformers/models/mellum/modeling_mellum.py +++ b/src/transformers/models/mellum/modeling_mellum.py @@ -355,6 +355,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return final_hidden_states.reshape(batch_size, sequence_length, hidden_dim) +@use_kernel_forward_from_hub("SwiGLUMLP") class MellumMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py b/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py index b9ad943f2507..c786870171f7 100644 --- a/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py +++ b/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py @@ -237,6 +237,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.experts(hidden_states, topk_indices, topk_weights).view(*orig_shape) +@use_kernel_forward_from_hub("SwiGLUMLP") class MiMoV2FlashMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/minicpm3/modeling_minicpm3.py b/src/transformers/models/minicpm3/modeling_minicpm3.py index 5d7e27a6cd92..b6c5b81c6b3a 100644 --- a/src/transformers/models/minicpm3/modeling_minicpm3.py +++ b/src/transformers/models/minicpm3/modeling_minicpm3.py @@ -134,6 +134,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class MiniCPM3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/ministral/modeling_ministral.py b/src/transformers/models/ministral/modeling_ministral.py index febf238729ba..423a08d45cda 100644 --- a/src/transformers/models/ministral/modeling_ministral.py +++ b/src/transformers/models/ministral/modeling_ministral.py @@ -47,6 +47,7 @@ from .configuration_ministral import MinistralConfig +@use_kernel_forward_from_hub("SwiGLUMLP") class MinistralMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/ministral3/modeling_ministral3.py b/src/transformers/models/ministral3/modeling_ministral3.py index 7bb5fdb0cc62..d5fa34c8787f 100644 --- a/src/transformers/models/ministral3/modeling_ministral3.py +++ b/src/transformers/models/ministral3/modeling_ministral3.py @@ -173,6 +173,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Ministral3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/mistral/modeling_mistral.py b/src/transformers/models/mistral/modeling_mistral.py index 6e3333148323..19231ec11f2f 100644 --- a/src/transformers/models/mistral/modeling_mistral.py +++ b/src/transformers/models/mistral/modeling_mistral.py @@ -32,6 +32,7 @@ from .configuration_mistral import MistralConfig +@use_kernel_forward_from_hub("SwiGLUMLP") class MistralMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/mistral4/modeling_mistral4.py b/src/transformers/models/mistral4/modeling_mistral4.py index b8c095868f96..89b1b13b0ae0 100644 --- a/src/transformers/models/mistral4/modeling_mistral4.py +++ b/src/transformers/models/mistral4/modeling_mistral4.py @@ -125,6 +125,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class Mistral4MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py index 5f14849afbbd..6de4add1072a 100644 --- a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py +++ b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py @@ -28,6 +28,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache, EncoderDecoderCache from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_bidirectional_mask, create_causal_mask from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_layers import GradientCheckpointingLayer @@ -431,6 +432,7 @@ def forward( return MoonshineStreamingEncoderModelOutput(last_hidden_state=hidden_states, attention_mask=attention_mask) +@use_kernel_forward_from_hub("SwiGLUMLP") class MoonshinMoonshineStreamingDecoderMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py b/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py index e44033281068..49dd8fcc344c 100644 --- a/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py +++ b/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py @@ -141,6 +141,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" +@use_kernel_forward_from_hub("GeGLUMLP") class MuseGlimmerTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py b/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py index 71e6c71f761d..da018b1379f0 100644 --- a/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py +++ b/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py @@ -217,6 +217,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class MuseGlimmerAssistantMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/nomic_bert/modeling_nomic_bert.py b/src/transformers/models/nomic_bert/modeling_nomic_bert.py index 83ff250bbe2d..d521ee152882 100644 --- a/src/transformers/models/nomic_bert/modeling_nomic_bert.py +++ b/src/transformers/models/nomic_bert/modeling_nomic_bert.py @@ -262,6 +262,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class NomicBertMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/nomic_bert/modular_nomic_bert.py b/src/transformers/models/nomic_bert/modular_nomic_bert.py index bd27e8034b62..17f88c9d94fd 100644 --- a/src/transformers/models/nomic_bert/modular_nomic_bert.py +++ b/src/transformers/models/nomic_bert/modular_nomic_bert.py @@ -19,6 +19,7 @@ from torch.nn import CrossEntropyLoss from ...configuration_utils import PreTrainedConfig +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_bidirectional_mask from ...modeling_outputs import ( BaseModelOutputWithPooling, @@ -117,6 +118,7 @@ def __init__(self, config): self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False) +@use_kernel_forward_from_hub("SwiGLUMLP") class NomicBertMLP(GemmaMLP): pass diff --git a/src/transformers/models/olmo/modeling_olmo.py b/src/transformers/models/olmo/modeling_olmo.py index dd2f1ed261c3..fe80867c1468 100644 --- a/src/transformers/models/olmo/modeling_olmo.py +++ b/src/transformers/models/olmo/modeling_olmo.py @@ -60,6 +60,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ) +@use_kernel_forward_from_hub("SwiGLUMLP") class OlmoMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmo2/modeling_olmo2.py b/src/transformers/models/olmo2/modeling_olmo2.py index 3a18aebc13b3..0ebc12da6265 100644 --- a/src/transformers/models/olmo2/modeling_olmo2.py +++ b/src/transformers/models/olmo2/modeling_olmo2.py @@ -266,6 +266,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Olmo2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmo3/modeling_olmo3.py b/src/transformers/models/olmo3/modeling_olmo3.py index 85a4317f683f..4371d333f7df 100644 --- a/src/transformers/models/olmo3/modeling_olmo3.py +++ b/src/transformers/models/olmo3/modeling_olmo3.py @@ -210,6 +210,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Olmo3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py b/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py index a54404a73ed8..999accf704e0 100644 --- a/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py +++ b/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py @@ -684,6 +684,7 @@ def forward( return output +@use_kernel_forward_from_hub("SwiGLUMLP") class OlmoHybridMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmoe/modeling_olmoe.py b/src/transformers/models/olmoe/modeling_olmoe.py index ab732636f8d3..aeaf9b7b9886 100644 --- a/src/transformers/models/olmoe/modeling_olmoe.py +++ b/src/transformers/models/olmoe/modeling_olmoe.py @@ -118,6 +118,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class OlmoeMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmoe/modular_olmoe.py b/src/transformers/models/olmoe/modular_olmoe.py index adadcf0c9e5d..ee0ed8a63c1d 100644 --- a/src/transformers/models/olmoe/modular_olmoe.py +++ b/src/transformers/models/olmoe/modular_olmoe.py @@ -19,6 +19,7 @@ from ... import initialization as init from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask from ...modeling_outputs import MoeModelOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel @@ -51,6 +52,7 @@ class OlmoeRotaryEmbedding(LlamaRotaryEmbedding): pass +@use_kernel_forward_from_hub("SwiGLUMLP") class OlmoeMLP(GemmaMLP): pass diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index e489ccf5b446..cdc2d544f032 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -124,6 +124,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Ovis2VisionMLP(nn.Module): def __init__(self, config): super().__init__() @@ -254,6 +255,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Ovis2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 115c4658dc9d..cc3fd94f5372 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -175,6 +175,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class PaddleOCRMLP(nn.Module): def __init__(self, config: PaddleOCRTextConfig): super().__init__() diff --git a/src/transformers/models/pe_audio/modeling_pe_audio.py b/src/transformers/models/pe_audio/modeling_pe_audio.py index c08e02ba5a4a..4456d163b263 100644 --- a/src/transformers/models/pe_audio/modeling_pe_audio.py +++ b/src/transformers/models/pe_audio/modeling_pe_audio.py @@ -431,6 +431,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class PeAudioEncoderMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py index b2a5d96df50a..072efaae2b8e 100644 --- a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py +++ b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py @@ -368,6 +368,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class PeAudioVideoEncoderMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index c1726eb8aa0d..7c463a62ecba 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -355,6 +355,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class PeVideoEncoderMLP(nn.Module): def __init__(self, config): super().__init__() From 741dba04c7c77612a41140bd43f4c790be8ce6ac Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 26 Aug 2026 11:08:44 +0000 Subject: [PATCH 4/8] last round --- src/transformers/models/qwen2/modeling_qwen2.py | 1 + src/transformers/models/qwen2_moe/modeling_qwen2_moe.py | 1 + src/transformers/models/qwen2_moe/modular_qwen2_moe.py | 2 ++ src/transformers/models/qwen3/modeling_qwen3.py | 1 + src/transformers/models/qwen3/modular_qwen3.py | 2 ++ src/transformers/models/qwen3_5/modeling_qwen3_5.py | 1 + src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py | 1 + src/transformers/models/qwen3_moe/modeling_qwen3_moe.py | 1 + src/transformers/models/qwen3_next/modeling_qwen3_next.py | 1 + .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 4 ++++ src/transformers/models/qwen3_vl/modeling_qwen3_vl.py | 1 + src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 1 + src/transformers/models/smollm3/modeling_smollm3.py | 1 + src/transformers/models/solar_open/modeling_solar_open.py | 1 + src/transformers/models/step3p7/modular_step3p7.py | 1 + src/transformers/models/t5gemma/modeling_t5gemma.py | 1 + src/transformers/models/t5gemma2/modeling_t5gemma2.py | 1 + src/transformers/models/vaultgemma/modeling_vaultgemma.py | 1 + .../models/voxtral_realtime/modeling_voxtral_realtime.py | 2 ++ src/transformers/models/youtu/modeling_youtu.py | 1 + 20 files changed, 26 insertions(+) diff --git a/src/transformers/models/qwen2/modeling_qwen2.py b/src/transformers/models/qwen2/modeling_qwen2.py index b4fed98fafae..422b00a020f2 100644 --- a/src/transformers/models/qwen2/modeling_qwen2.py +++ b/src/transformers/models/qwen2/modeling_qwen2.py @@ -32,6 +32,7 @@ from .configuration_qwen2 import Qwen2Config +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py index 330ac6404a3e..86a6b5b66207 100644 --- a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py @@ -127,6 +127,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen2MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen2_moe/modular_qwen2_moe.py b/src/transformers/models/qwen2_moe/modular_qwen2_moe.py index cb46349cbfb3..cbc21d95c032 100644 --- a/src/transformers/models/qwen2_moe/modular_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modular_qwen2_moe.py @@ -25,6 +25,7 @@ from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...generation import GenerationMixin +from ...integrations import use_kernel_forward_from_hub from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask from ...modeling_layers import ( GenericForQuestionAnswering, @@ -56,6 +57,7 @@ class Qwen2MoeRotaryEmbedding(Gemma2RotaryEmbedding): pass +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen2MoeMLP(GemmaMLP): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen3/modeling_qwen3.py b/src/transformers/models/qwen3/modeling_qwen3.py index 5e474983ab92..1d7d0d237eb9 100644 --- a/src/transformers/models/qwen3/modeling_qwen3.py +++ b/src/transformers/models/qwen3/modeling_qwen3.py @@ -67,6 +67,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen3/modular_qwen3.py b/src/transformers/models/qwen3/modular_qwen3.py index ea6ddb2f7499..5082bd817d58 100644 --- a/src/transformers/models/qwen3/modular_qwen3.py +++ b/src/transformers/models/qwen3/modular_qwen3.py @@ -18,6 +18,7 @@ import torch from ...cache_utils import Cache +from ...integrations import use_kernel_forward_from_hub from ...modeling_flash_attention_utils import FlashAttentionKwargs from ...modeling_outputs import CausalLMOutputWithPast from ...modeling_utils import ALL_ATTENTION_FUNCTIONS @@ -49,6 +50,7 @@ class Qwen3RMSNorm(Qwen2RMSNorm): pass +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3MLP(GemmaMLP): pass diff --git a/src/transformers/models/qwen3_5/modeling_qwen3_5.py b/src/transformers/models/qwen3_5/modeling_qwen3_5.py index 1ecd206ecda3..37d3394e80f7 100644 --- a/src/transformers/models/qwen3_5/modeling_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modeling_qwen3_5.py @@ -702,6 +702,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3_5MLP(nn.Module): def __init__(self, config: Qwen3_5Config, intermediate_size: int): super().__init__() diff --git a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py index 3243b69b4156..c1b5d1e51bd6 100644 --- a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py +++ b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py @@ -702,6 +702,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3_5MoeMLP(nn.Module): def __init__(self, config: Qwen3_5MoeConfig, intermediate_size: int): super().__init__() diff --git a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py index a4b13c545601..0f706aad1257 100644 --- a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py @@ -190,6 +190,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen3_next/modeling_qwen3_next.py b/src/transformers/models/qwen3_next/modeling_qwen3_next.py index 9889d4f3cd5f..03faac94f752 100644 --- a/src/transformers/models/qwen3_next/modeling_qwen3_next.py +++ b/src/transformers/models/qwen3_next/modeling_qwen3_next.py @@ -699,6 +699,7 @@ def forward( return output +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3NextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index af037ffbb664..986e4edfdcee 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1499,6 +1499,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3OmniMoeThinkerTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() @@ -2402,6 +2403,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3OmniMoeMLP(nn.Module): def __init__(self, config): super().__init__() @@ -2713,6 +2715,7 @@ class Qwen3OmniMoeTalkerRotaryEmbedding(Qwen3OmniMoeThinkerTextRotaryEmbedding): pass +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3OmniMoeTalkerTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() @@ -3417,6 +3420,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3OmniMoeCode2WavMlp(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 14e8372647ac..9a8bb31cf01a 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -507,6 +507,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3VLTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index b1834f545af8..e54deef8dba4 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -303,6 +303,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen3VLMoeTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/smollm3/modeling_smollm3.py b/src/transformers/models/smollm3/modeling_smollm3.py index fcf2eac3556e..eafa66416bdb 100644 --- a/src/transformers/models/smollm3/modeling_smollm3.py +++ b/src/transformers/models/smollm3/modeling_smollm3.py @@ -268,6 +268,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class SmolLM3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/solar_open/modeling_solar_open.py b/src/transformers/models/solar_open/modeling_solar_open.py index e685d8848892..d4c40a1227d3 100644 --- a/src/transformers/models/solar_open/modeling_solar_open.py +++ b/src/transformers/models/solar_open/modeling_solar_open.py @@ -83,6 +83,7 @@ def forward( return hidden_states +@use_kernel_forward_from_hub("SwiGLUMLP") class SolarOpenMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/step3p7/modular_step3p7.py b/src/transformers/models/step3p7/modular_step3p7.py index 7b3390980bae..023f3571895f 100644 --- a/src/transformers/models/step3p7/modular_step3p7.py +++ b/src/transformers/models/step3p7/modular_step3p7.py @@ -784,6 +784,7 @@ class Step3p7RMSNorm(MiniMaxM3VLRMSNorm): pass +@no_inherit_decorator class Step3p7MLP(DeepseekV4MLP): def __init__(self, config, layer_idx, is_shared_expert=False): super().__init__(config) diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index ddbacedb7782..18edb558523b 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -78,6 +78,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" +@use_kernel_forward_from_hub("GeGLUMLP") class T5GemmaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index d54e95d76a59..44b81e5bd480 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -72,6 +72,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" +@use_kernel_forward_from_hub("GeGLUMLP") class T5Gemma2MLP(nn.Module): def __init__(self, config: T5Gemma2TextConfig): super().__init__() diff --git a/src/transformers/models/vaultgemma/modeling_vaultgemma.py b/src/transformers/models/vaultgemma/modeling_vaultgemma.py index ad84fd75c125..13ae2b5e841a 100644 --- a/src/transformers/models/vaultgemma/modeling_vaultgemma.py +++ b/src/transformers/models/vaultgemma/modeling_vaultgemma.py @@ -63,6 +63,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" +@use_kernel_forward_from_hub("GeGLUMLP") class VaultGemmaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py index e3abf0c60b9b..5d62eb163201 100644 --- a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py +++ b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py @@ -396,6 +396,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class VoxtralRealtimeMLP(nn.Module): def __init__(self, config): super().__init__() @@ -681,6 +682,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class VoxtralRealtimeTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/youtu/modeling_youtu.py b/src/transformers/models/youtu/modeling_youtu.py index 7292fc523c48..ecefe609a48a 100644 --- a/src/transformers/models/youtu/modeling_youtu.py +++ b/src/transformers/models/youtu/modeling_youtu.py @@ -127,6 +127,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) +@use_kernel_forward_from_hub("SwiGLUMLP") class YoutuMLP(nn.Module): def __init__(self, config): super().__init__() From 23a89a77e9bfef4b3b361245ff512c65d5bf6cef Mon Sep 17 00:00:00 2001 From: vasqu Date: Wed, 26 Aug 2026 13:45:39 +0000 Subject: [PATCH 5/8] fix these models --- src/transformers/models/inkling/modeling_inkling.py | 1 - src/transformers/models/inkling/modular_inkling.py | 3 ++- src/transformers/models/t5gemma/modeling_t5gemma.py | 1 - src/transformers/models/t5gemma/modular_t5gemma.py | 3 ++- src/transformers/models/t5gemma2/modeling_t5gemma2.py | 1 - src/transformers/models/t5gemma2/modular_t5gemma2.py | 3 ++- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/inkling/modeling_inkling.py b/src/transformers/models/inkling/modeling_inkling.py index 017ec2683c59..0bb061b3ea37 100644 --- a/src/transformers/models/inkling/modeling_inkling.py +++ b/src/transformers/models/inkling/modeling_inkling.py @@ -282,7 +282,6 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("GeGLUMLP") class InklingMLP(nn.Module): def __init__(self, config: InklingTextConfig): super().__init__() diff --git a/src/transformers/models/inkling/modular_inkling.py b/src/transformers/models/inkling/modular_inkling.py index 89276fa3128c..1cdc7ac80282 100644 --- a/src/transformers/models/inkling/modular_inkling.py +++ b/src/transformers/models/inkling/modular_inkling.py @@ -40,7 +40,7 @@ logging, torch_compilable_check, ) -from ...utils.generic import merge_with_config_defaults +from ...utils.generic import merge_with_config_defaults, no_inherit_decorator from ...utils.output_capturing import capture_outputs from ..gemma3.modeling_gemma3 import ( Gemma3CausalLMOutputWithPast, @@ -500,6 +500,7 @@ def forward( return attn_output, attn_weights +@no_inherit_decorator class InklingMLP(Gemma3MLP): def __init__(self, config: InklingTextConfig): super().__init__(config) diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index 18edb558523b..ddbacedb7782 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -78,7 +78,6 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" -@use_kernel_forward_from_hub("GeGLUMLP") class T5GemmaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/t5gemma/modular_t5gemma.py b/src/transformers/models/t5gemma/modular_t5gemma.py index 6f0c399830e5..c686e19a6c7f 100644 --- a/src/transformers/models/t5gemma/modular_t5gemma.py +++ b/src/transformers/models/t5gemma/modular_t5gemma.py @@ -48,7 +48,7 @@ can_return_tuple, logging, ) -from ...utils.generic import merge_with_config_defaults +from ...utils.generic import merge_with_config_defaults, no_inherit_decorator from ...utils.output_capturing import OutputRecorder, capture_outputs from ..gemma2.configuration_gemma2 import Gemma2Config from ..gemma2.modeling_gemma2 import ( @@ -153,6 +153,7 @@ class T5GemmaRMSNorm(Gemma2RMSNorm): pass +@no_inherit_decorator class T5GemmaMLP(Gemma2MLP): def __init__(self, config): super().__init__(config) diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index 44b81e5bd480..d54e95d76a59 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -72,7 +72,6 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" -@use_kernel_forward_from_hub("GeGLUMLP") class T5Gemma2MLP(nn.Module): def __init__(self, config: T5Gemma2TextConfig): super().__init__() diff --git a/src/transformers/models/t5gemma2/modular_t5gemma2.py b/src/transformers/models/t5gemma2/modular_t5gemma2.py index 131106686fca..4f6ada91a2dc 100644 --- a/src/transformers/models/t5gemma2/modular_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modular_t5gemma2.py @@ -45,7 +45,7 @@ logging, torch_compilable_check, ) -from ...utils.generic import merge_with_config_defaults +from ...utils.generic import merge_with_config_defaults, no_inherit_decorator from ...utils.output_capturing import OutputRecorder, capture_outputs from ..auto import AutoModel from ..gemma3.configuration_gemma3 import Gemma3Config, Gemma3TextConfig @@ -231,6 +231,7 @@ class T5Gemma2RMSNorm(Gemma3RMSNorm): pass +@no_inherit_decorator class T5Gemma2MLP(Gemma3MLP): def __init__(self, config: T5Gemma2TextConfig): super().__init__(config) From ad6a6c0787a3f6e753e45b00fec09e0c6738afaf Mon Sep 17 00:00:00 2001 From: vasqu Date: Mon, 31 Aug 2026 19:04:46 +0000 Subject: [PATCH 6/8] after merge sync --- src/transformers/models/glm5_next/modeling_glm5_next.py | 1 + src/transformers/models/qwen4_exp/modeling_qwen4_exp.py | 1 + src/transformers/models/vibevoice/modeling_vibevoice.py | 1 + 3 files changed, 3 insertions(+) diff --git a/src/transformers/models/glm5_next/modeling_glm5_next.py b/src/transformers/models/glm5_next/modeling_glm5_next.py index ba0551f62ad8..845b50a418e1 100644 --- a/src/transformers/models/glm5_next/modeling_glm5_next.py +++ b/src/transformers/models/glm5_next/modeling_glm5_next.py @@ -82,6 +82,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" +@use_kernel_forward_from_hub("SwiGLUMLP") class Glm5NextTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py b/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py index 4273886cc4ab..b07bede3e130 100644 --- a/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py +++ b/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py @@ -912,6 +912,7 @@ def forward( return attn_output, attn_weights +@use_kernel_forward_from_hub("SwiGLUMLP") class Qwen4ExpTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/vibevoice/modeling_vibevoice.py b/src/transformers/models/vibevoice/modeling_vibevoice.py index 77afa1d29ca3..0e068bff70e6 100644 --- a/src/transformers/models/vibevoice/modeling_vibevoice.py +++ b/src/transformers/models/vibevoice/modeling_vibevoice.py @@ -118,6 +118,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states +@use_kernel_forward_from_hub("SwiGLUMLP") class VibeVoiceMLP(nn.Module): def __init__(self, config): super().__init__() From d16cbc9ed782bf9c21251eae1cae990406ab4931 Mon Sep 17 00:00:00 2001 From: vasqu Date: Mon, 31 Aug 2026 19:28:49 +0000 Subject: [PATCH 7/8] add condition --- src/transformers/integrations/hub_kernels.py | 4 ++-- src/transformers/models/afmoe/modeling_afmoe.py | 2 +- src/transformers/models/aimv2/modeling_aimv2.py | 2 +- src/transformers/models/aria/modeling_aria.py | 2 +- src/transformers/models/axk1/modeling_axk1.py | 2 +- src/transformers/models/axk2/modeling_axk2.py | 2 +- src/transformers/models/bamba/modeling_bamba.py | 2 +- src/transformers/models/cohere/modeling_cohere.py | 2 +- src/transformers/models/cohere2/modeling_cohere2.py | 2 +- .../models/cohere2_moe/modeling_cohere2_moe.py | 2 +- .../models/cohere_compass/modeling_cohere_compass.py | 2 +- src/transformers/models/csm/modeling_csm.py | 2 +- src/transformers/models/cwm/modeling_cwm.py | 2 +- .../models/deepseek_ocr2/modeling_deepseek_ocr2.py | 2 +- .../models/deepseek_v2/modeling_deepseek_v2.py | 2 +- .../models/deepseek_v3/modeling_deepseek_v3.py | 2 +- .../models/deepseek_v32/modeling_deepseek_v32.py | 2 +- .../models/deepseek_v4/modeling_deepseek_v4.py | 2 +- src/transformers/models/deimv2/modeling_deimv2.py | 2 +- src/transformers/models/diffllama/modeling_diffllama.py | 2 +- .../models/diffusion_gemma/modeling_diffusion_gemma.py | 2 +- src/transformers/models/doge/modeling_doge.py | 2 +- src/transformers/models/dots1/modeling_dots1.py | 2 +- src/transformers/models/emu3/modeling_emu3.py | 2 +- src/transformers/models/ernie4_5/modeling_ernie4_5.py | 2 +- .../models/ernie4_5_moe/modeling_ernie4_5_moe.py | 2 +- .../models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py | 2 +- src/transformers/models/esmc/modeling_esmc.py | 2 +- src/transformers/models/eurobert/modeling_eurobert.py | 2 +- src/transformers/models/evolla/modeling_evolla.py | 2 +- src/transformers/models/exaone4/modeling_exaone4.py | 2 +- src/transformers/models/exaone_moe/modeling_exaone_moe.py | 2 +- src/transformers/models/falcon_h1/modeling_falcon_h1.py | 2 +- src/transformers/models/flex_olmo/modeling_flex_olmo.py | 2 +- src/transformers/models/gemma/modeling_gemma.py | 2 +- src/transformers/models/gemma/modular_gemma.py | 2 +- src/transformers/models/gemma2/modeling_gemma2.py | 2 +- src/transformers/models/gemma3/modeling_gemma3.py | 2 +- src/transformers/models/gemma4/modeling_gemma4.py | 4 ++-- .../models/gemma4_unified/modeling_gemma4_unified.py | 2 +- src/transformers/models/glm4_moe/modeling_glm4_moe.py | 2 +- .../models/glm4_moe_lite/modeling_glm4_moe_lite.py | 2 +- src/transformers/models/glm4v_moe/modeling_glm4v_moe.py | 2 +- src/transformers/models/glm5_next/modeling_glm5_next.py | 2 +- .../models/glm_moe_dsa/modeling_glm_moe_dsa.py | 2 +- src/transformers/models/granite/modeling_granite.py | 2 +- .../models/granite4_vision/modeling_granite4_vision.py | 2 +- .../models/granite_swa/modeling_granite_swa.py | 2 +- src/transformers/models/helium/modeling_helium.py | 2 +- .../models/higgs_audio_v2/modeling_higgs_audio_v2.py | 2 +- src/transformers/models/hrm_text/modeling_hrm_text.py | 2 +- .../models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py | 2 +- .../models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py | 2 +- src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py | 2 +- src/transformers/models/hy_v3/modeling_hy_v3.py | 2 +- .../models/hyperclovax/modeling_hyperclovax.py | 2 +- src/transformers/models/jamba/modeling_jamba.py | 2 +- src/transformers/models/laguna/modeling_laguna.py | 2 +- src/transformers/models/llama/modeling_llama.py | 2 +- .../models/longcat_flash/modeling_longcat_flash.py | 2 +- src/transformers/models/mellum/modeling_mellum.py | 2 +- .../models/mimo_v2_flash/modeling_mimo_v2_flash.py | 2 +- src/transformers/models/minicpm3/modeling_minicpm3.py | 2 +- src/transformers/models/ministral/modeling_ministral.py | 2 +- src/transformers/models/ministral3/modeling_ministral3.py | 2 +- src/transformers/models/mistral/modeling_mistral.py | 2 +- src/transformers/models/mistral4/modeling_mistral4.py | 2 +- .../moonshine_streaming/modeling_moonshine_streaming.py | 2 +- .../models/muse_glimmer/modeling_muse_glimmer.py | 2 +- .../modeling_muse_glimmer_assistant.py | 2 +- src/transformers/models/nomic_bert/modeling_nomic_bert.py | 2 +- src/transformers/models/nomic_bert/modular_nomic_bert.py | 2 +- src/transformers/models/olmo/modeling_olmo.py | 2 +- src/transformers/models/olmo2/modeling_olmo2.py | 2 +- src/transformers/models/olmo3/modeling_olmo3.py | 2 +- .../models/olmo_hybrid/modeling_olmo_hybrid.py | 2 +- src/transformers/models/olmoe/modeling_olmoe.py | 2 +- src/transformers/models/olmoe/modular_olmoe.py | 2 +- src/transformers/models/ovis2/modeling_ovis2.py | 4 ++-- .../models/paddleocr_vl/modeling_paddleocr_vl.py | 2 +- src/transformers/models/pe_audio/modeling_pe_audio.py | 2 +- .../models/pe_audio_video/modeling_pe_audio_video.py | 2 +- src/transformers/models/pe_video/modeling_pe_video.py | 2 +- src/transformers/models/qwen2/modeling_qwen2.py | 2 +- src/transformers/models/qwen2_moe/modeling_qwen2_moe.py | 2 +- src/transformers/models/qwen2_moe/modular_qwen2_moe.py | 2 +- src/transformers/models/qwen3/modeling_qwen3.py | 2 +- src/transformers/models/qwen3/modular_qwen3.py | 2 +- src/transformers/models/qwen3_5/modeling_qwen3_5.py | 2 +- .../models/qwen3_5_moe/modeling_qwen3_5_moe.py | 2 +- src/transformers/models/qwen3_moe/modeling_qwen3_moe.py | 2 +- src/transformers/models/qwen3_next/modeling_qwen3_next.py | 2 +- .../models/qwen3_omni_moe/modeling_qwen3_omni_moe.py | 8 ++++---- src/transformers/models/qwen3_vl/modeling_qwen3_vl.py | 2 +- .../models/qwen3_vl_moe/modeling_qwen3_vl_moe.py | 2 +- src/transformers/models/qwen4_exp/modeling_qwen4_exp.py | 2 +- src/transformers/models/smollm3/modeling_smollm3.py | 2 +- src/transformers/models/solar_open/modeling_solar_open.py | 2 +- src/transformers/models/vaultgemma/modeling_vaultgemma.py | 2 +- src/transformers/models/vibevoice/modeling_vibevoice.py | 2 +- .../models/voxtral_realtime/modeling_voxtral_realtime.py | 4 ++-- src/transformers/models/youtu/modeling_youtu.py | 2 +- 102 files changed, 109 insertions(+), 109 deletions(-) diff --git a/src/transformers/integrations/hub_kernels.py b/src/transformers/integrations/hub_kernels.py index b58adc35c063..0b89aafb2457 100644 --- a/src/transformers/integrations/hub_kernels.py +++ b/src/transformers/integrations/hub_kernels.py @@ -98,9 +98,9 @@ ) from kernels import use_kernelized_func as _kernels_use_kernelized_func - def use_kernel_forward_from_hub(layer_name: str): + def use_kernel_forward_from_hub(layer_name: str, condition: Callable[["nn.Module"], bool] | None = None): if _kernels_enabled: - return _kernels_use_kernel_forward_from_hub(layer_name) + return _kernels_use_kernel_forward_from_hub(layer_name, condition) else: logger.warning_once( f"kernels hub usage is disabled through the environment USE_HUB_KERNELS={_TRANSFORMERS_USE_HUB_KERNELS}" diff --git a/src/transformers/models/afmoe/modeling_afmoe.py b/src/transformers/models/afmoe/modeling_afmoe.py index 69595083a045..fc8fc35db66c 100644 --- a/src/transformers/models/afmoe/modeling_afmoe.py +++ b/src/transformers/models/afmoe/modeling_afmoe.py @@ -119,7 +119,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class AfmoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/aimv2/modeling_aimv2.py b/src/transformers/models/aimv2/modeling_aimv2.py index ae6854c3a730..8557e3611650 100644 --- a/src/transformers/models/aimv2/modeling_aimv2.py +++ b/src/transformers/models/aimv2/modeling_aimv2.py @@ -97,7 +97,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Aimv2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/aria/modeling_aria.py b/src/transformers/models/aria/modeling_aria.py index 0d83fe4991bd..6a29680de498 100644 --- a/src/transformers/models/aria/modeling_aria.py +++ b/src/transformers/models/aria/modeling_aria.py @@ -215,7 +215,7 @@ def forward(self, key_value_states: torch.Tensor, attn_mask: torch.Tensor | None return out -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class AriaSharedExpertsMLP(nn.Module): """ Shared Expert MLP for shared experts. diff --git a/src/transformers/models/axk1/modeling_axk1.py b/src/transformers/models/axk1/modeling_axk1.py index c015dd8b3497..14075e5cd4b1 100644 --- a/src/transformers/models/axk1/modeling_axk1.py +++ b/src/transformers/models/axk1/modeling_axk1.py @@ -126,7 +126,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class AXK1MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/axk2/modeling_axk2.py b/src/transformers/models/axk2/modeling_axk2.py index 05f29ea9e626..9b8ebed35bab 100644 --- a/src/transformers/models/axk2/modeling_axk2.py +++ b/src/transformers/models/axk2/modeling_axk2.py @@ -381,7 +381,7 @@ def forward( return final_hidden_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class AXK2MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/bamba/modeling_bamba.py b/src/transformers/models/bamba/modeling_bamba.py index 46e8959020f9..0f9a180100ac 100644 --- a/src/transformers/models/bamba/modeling_bamba.py +++ b/src/transformers/models/bamba/modeling_bamba.py @@ -795,7 +795,7 @@ def forward( return contextualized_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class BambaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/cohere/modeling_cohere.py b/src/transformers/models/cohere/modeling_cohere.py index d071076dad82..1bfe3d55ce2b 100644 --- a/src/transformers/models/cohere/modeling_cohere.py +++ b/src/transformers/models/cohere/modeling_cohere.py @@ -120,7 +120,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class CohereMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/cohere2/modeling_cohere2.py b/src/transformers/models/cohere2/modeling_cohere2.py index 2843c0eb60d2..00e572e13bad 100644 --- a/src/transformers/models/cohere2/modeling_cohere2.py +++ b/src/transformers/models/cohere2/modeling_cohere2.py @@ -254,7 +254,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Cohere2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py b/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py index 6b4012fae146..f66ead99d97a 100644 --- a/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py +++ b/src/transformers/models/cohere2_moe/modeling_cohere2_moe.py @@ -82,7 +82,7 @@ def forward(self, hidden_states): return hidden_states.to(input_dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Cohere2MoeMLP(nn.Module): def __init__(self, config: Cohere2MoeConfig, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/cohere_compass/modeling_cohere_compass.py b/src/transformers/models/cohere_compass/modeling_cohere_compass.py index fa1fbbd99c2c..035eaabdbd71 100644 --- a/src/transformers/models/cohere_compass/modeling_cohere_compass.py +++ b/src/transformers/models/cohere_compass/modeling_cohere_compass.py @@ -160,7 +160,7 @@ def apply_interleaved_mrope(self, freqs, mrope_section): return freqs_t -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class CohereCompassMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/csm/modeling_csm.py b/src/transformers/models/csm/modeling_csm.py index ff61e41f1695..8f84fd81d525 100644 --- a/src/transformers/models/csm/modeling_csm.py +++ b/src/transformers/models/csm/modeling_csm.py @@ -174,7 +174,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class CsmMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/cwm/modeling_cwm.py b/src/transformers/models/cwm/modeling_cwm.py index c9751da9cebe..f67d1fbe17ec 100644 --- a/src/transformers/models/cwm/modeling_cwm.py +++ b/src/transformers/models/cwm/modeling_cwm.py @@ -249,7 +249,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class CwmMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py b/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py index ad320db831cf..0d17be2f5c44 100644 --- a/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py +++ b/src/transformers/models/deepseek_ocr2/modeling_deepseek_ocr2.py @@ -1118,7 +1118,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class DeepseekOcr2TextMLP(nn.Module): def __init__(self, config: DeepseekOcr2TextConfig, hidden_size=None, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py index 665fc2aa7e04..6551549bb915 100644 --- a/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py +++ b/src/transformers/models/deepseek_v2/modeling_deepseek_v2.py @@ -137,7 +137,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class DeepseekV2MLP(nn.Module): def __init__(self, config: DeepseekV2Config, hidden_size=None, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py index 9fe62045cbfa..338db92101f0 100644 --- a/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py +++ b/src/transformers/models/deepseek_v3/modeling_deepseek_v3.py @@ -112,7 +112,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class DeepseekV3MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py b/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py index 1ab9a72dc6b7..71f029483544 100644 --- a/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py +++ b/src/transformers/models/deepseek_v32/modeling_deepseek_v32.py @@ -503,7 +503,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class DeepseekV32MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py b/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py index 745b3ea2bc6e..d500fe2158a9 100644 --- a/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py +++ b/src/transformers/models/deepseek_v4/modeling_deepseek_v4.py @@ -962,7 +962,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return (pre.unsqueeze(-1) * x).sum(dim=2).to(x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class DeepseekV4MLP(nn.Module): def __init__(self, config: DeepseekV4Config): super().__init__() diff --git a/src/transformers/models/deimv2/modeling_deimv2.py b/src/transformers/models/deimv2/modeling_deimv2.py index d881b94719da..57676d5de5c5 100644 --- a/src/transformers/models/deimv2/modeling_deimv2.py +++ b/src/transformers/models/deimv2/modeling_deimv2.py @@ -177,7 +177,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Deimv2SwiGLUFFN(nn.Module): def __init__(self, config: Deimv2Config): super().__init__() diff --git a/src/transformers/models/diffllama/modeling_diffllama.py b/src/transformers/models/diffllama/modeling_diffllama.py index 54356c80dc54..4f9b5f4c658e 100644 --- a/src/transformers/models/diffllama/modeling_diffllama.py +++ b/src/transformers/models/diffllama/modeling_diffllama.py @@ -49,7 +49,7 @@ from .configuration_diffllama import DiffLlamaConfig -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class DiffLlamaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py b/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py index 8580fd6a985d..02c205b63909 100644 --- a/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py +++ b/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py @@ -497,7 +497,7 @@ def append_to_cache( return keys, values -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class DiffusionGemmaText4MLP(nn.Module): def __init__(self, config: DiffusionGemmaTextConfig, layer_idx: int): super().__init__() diff --git a/src/transformers/models/doge/modeling_doge.py b/src/transformers/models/doge/modeling_doge.py index 3cfe429eb79c..58816798f072 100644 --- a/src/transformers/models/doge/modeling_doge.py +++ b/src/transformers/models/doge/modeling_doge.py @@ -371,7 +371,7 @@ def prepare_dynamic_mask( return attn_mask -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class DogeMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/dots1/modeling_dots1.py b/src/transformers/models/dots1/modeling_dots1.py index 502f5d2fbad5..6ad4d83368bc 100644 --- a/src/transformers/models/dots1/modeling_dots1.py +++ b/src/transformers/models/dots1/modeling_dots1.py @@ -260,7 +260,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Dots1MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/emu3/modeling_emu3.py b/src/transformers/models/emu3/modeling_emu3.py index a208ba21467f..bd6b37117a2e 100644 --- a/src/transformers/models/emu3/modeling_emu3.py +++ b/src/transformers/models/emu3/modeling_emu3.py @@ -216,7 +216,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Emu3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/ernie4_5/modeling_ernie4_5.py b/src/transformers/models/ernie4_5/modeling_ernie4_5.py index 0b0889be2f63..e4f381a68be9 100644 --- a/src/transformers/models/ernie4_5/modeling_ernie4_5.py +++ b/src/transformers/models/ernie4_5/modeling_ernie4_5.py @@ -95,7 +95,7 @@ def forward(self, x, position_ids): return cos, sin -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Ernie4_5MLP(nn.Module): def __init__(self, config: Ernie4_5Config): super().__init__() diff --git a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py index 9addf5c08c3a..91776332136b 100644 --- a/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py +++ b/src/transformers/models/ernie4_5_moe/modeling_ernie4_5_moe.py @@ -63,7 +63,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Ernie4_5_MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py index 48e8b7881a55..0835bcbe7f7a 100644 --- a/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py +++ b/src/transformers/models/ernie4_5_vl_moe/modeling_ernie4_5_vl_moe.py @@ -290,7 +290,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Ernie4_5_VLMoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/esmc/modeling_esmc.py b/src/transformers/models/esmc/modeling_esmc.py index 22015bbd8da4..5ed3be518c0f 100644 --- a/src/transformers/models/esmc/modeling_esmc.py +++ b/src/transformers/models/esmc/modeling_esmc.py @@ -110,7 +110,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class EsmcMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/eurobert/modeling_eurobert.py b/src/transformers/models/eurobert/modeling_eurobert.py index 1e4f96051278..700aa4d14a40 100644 --- a/src/transformers/models/eurobert/modeling_eurobert.py +++ b/src/transformers/models/eurobert/modeling_eurobert.py @@ -200,7 +200,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class EuroBertMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/evolla/modeling_evolla.py b/src/transformers/models/evolla/modeling_evolla.py index 67151b1abcf4..e732d299b18c 100644 --- a/src/transformers/models/evolla/modeling_evolla.py +++ b/src/transformers/models/evolla/modeling_evolla.py @@ -1083,7 +1083,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class EvollaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/exaone4/modeling_exaone4.py b/src/transformers/models/exaone4/modeling_exaone4.py index 15944ecfbba7..a6e778933de5 100644 --- a/src/transformers/models/exaone4/modeling_exaone4.py +++ b/src/transformers/models/exaone4/modeling_exaone4.py @@ -265,7 +265,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Exaone4MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/exaone_moe/modeling_exaone_moe.py b/src/transformers/models/exaone_moe/modeling_exaone_moe.py index 8567144fe242..70847e7f5e76 100644 --- a/src/transformers/models/exaone_moe/modeling_exaone_moe.py +++ b/src/transformers/models/exaone_moe/modeling_exaone_moe.py @@ -208,7 +208,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class ExaoneMoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/falcon_h1/modeling_falcon_h1.py b/src/transformers/models/falcon_h1/modeling_falcon_h1.py index 468e592e4838..bb7d19892f12 100644 --- a/src/transformers/models/falcon_h1/modeling_falcon_h1.py +++ b/src/transformers/models/falcon_h1/modeling_falcon_h1.py @@ -822,7 +822,7 @@ def forward( return contextualized_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class FalconH1MLP(nn.Module): def __init__(self, config: FalconH1Config): super().__init__() diff --git a/src/transformers/models/flex_olmo/modeling_flex_olmo.py b/src/transformers/models/flex_olmo/modeling_flex_olmo.py index 831c34aa9cec..8ac5b24fdb68 100644 --- a/src/transformers/models/flex_olmo/modeling_flex_olmo.py +++ b/src/transformers/models/flex_olmo/modeling_flex_olmo.py @@ -117,7 +117,7 @@ def forward(self, x, position_ids): return cos, sin -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class FlexOlmoMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/gemma/modeling_gemma.py b/src/transformers/models/gemma/modeling_gemma.py index 1b40b3ce1e96..520c232e9be0 100644 --- a/src/transformers/models/gemma/modeling_gemma.py +++ b/src/transformers/models/gemma/modeling_gemma.py @@ -81,7 +81,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class GemmaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/gemma/modular_gemma.py b/src/transformers/models/gemma/modular_gemma.py index b6c6b957aa15..879663989f74 100644 --- a/src/transformers/models/gemma/modular_gemma.py +++ b/src/transformers/models/gemma/modular_gemma.py @@ -137,7 +137,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class GemmaMLP(LlamaMLP): def __init__(self, config): super().__init__(config) diff --git a/src/transformers/models/gemma2/modeling_gemma2.py b/src/transformers/models/gemma2/modeling_gemma2.py index d55d708ba80e..5e53a363628d 100644 --- a/src/transformers/models/gemma2/modeling_gemma2.py +++ b/src/transformers/models/gemma2/modeling_gemma2.py @@ -66,7 +66,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class Gemma2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index 4649263243f2..c091cf5ccc76 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -117,7 +117,7 @@ def forward(self, input_ids: torch.Tensor): return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype) -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class Gemma3MLP(nn.Module): def __init__(self, config: Gemma3TextConfig): super().__init__() diff --git a/src/transformers/models/gemma4/modeling_gemma4.py b/src/transformers/models/gemma4/modeling_gemma4.py index 59de0df4491e..c2ea67780236 100644 --- a/src/transformers/models/gemma4/modeling_gemma4.py +++ b/src/transformers/models/gemma4/modeling_gemma4.py @@ -688,7 +688,7 @@ def forward( return hidden_states, padding_positions -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class Gemma4VisionMLP(nn.Module): def __init__(self, config: Gemma4VisionConfig): super().__init__() @@ -1063,7 +1063,7 @@ def forward( return BaseModelOutputWithPast(last_hidden_state=hidden_states) -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class Gemma4TextMLP(nn.Module): def __init__(self, config: Gemma4TextConfig, layer_idx: int): super().__init__() diff --git a/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py b/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py index 05836e7ea2b2..b9569afeb55d 100644 --- a/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py +++ b/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py @@ -456,7 +456,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class Gemma4UnifiedTextMLP(nn.Module): def __init__(self, config: Gemma4UnifiedTextConfig, layer_idx: int): super().__init__() diff --git a/src/transformers/models/glm4_moe/modeling_glm4_moe.py b/src/transformers/models/glm4_moe/modeling_glm4_moe.py index 29e7b729f793..ba591e80508d 100644 --- a/src/transformers/models/glm4_moe/modeling_glm4_moe.py +++ b/src/transformers/models/glm4_moe/modeling_glm4_moe.py @@ -260,7 +260,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Glm4MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py index 1ba436706fb0..bfb6e40434d5 100644 --- a/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py +++ b/src/transformers/models/glm4_moe_lite/modeling_glm4_moe_lite.py @@ -367,7 +367,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Glm4MoeLiteMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py index 2c919b1e7c77..37ea3f885f7d 100644 --- a/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py +++ b/src/transformers/models/glm4v_moe/modeling_glm4v_moe.py @@ -314,7 +314,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Glm4vMoeTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/glm5_next/modeling_glm5_next.py b/src/transformers/models/glm5_next/modeling_glm5_next.py index 845b50a418e1..08df37e518d9 100644 --- a/src/transformers/models/glm5_next/modeling_glm5_next.py +++ b/src/transformers/models/glm5_next/modeling_glm5_next.py @@ -82,7 +82,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Glm5NextTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py index 5d1bf1c331eb..6e4872e4f2e0 100644 --- a/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py +++ b/src/transformers/models/glm_moe_dsa/modeling_glm_moe_dsa.py @@ -480,7 +480,7 @@ def forward( return attn_output, attn_weights, topk_indices -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class GlmMoeDsaMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/granite/modeling_granite.py b/src/transformers/models/granite/modeling_granite.py index b1ebba4b1675..a5cec34c913b 100644 --- a/src/transformers/models/granite/modeling_granite.py +++ b/src/transformers/models/granite/modeling_granite.py @@ -200,7 +200,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class GraniteMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/granite4_vision/modeling_granite4_vision.py b/src/transformers/models/granite4_vision/modeling_granite4_vision.py index 89df07bafbf1..080ec479c2df 100644 --- a/src/transformers/models/granite4_vision/modeling_granite4_vision.py +++ b/src/transformers/models/granite4_vision/modeling_granite4_vision.py @@ -428,7 +428,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Granite4VisionTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/granite_swa/modeling_granite_swa.py b/src/transformers/models/granite_swa/modeling_granite_swa.py index c81eaa39366a..b384a68f52c1 100644 --- a/src/transformers/models/granite_swa/modeling_granite_swa.py +++ b/src/transformers/models/granite_swa/modeling_granite_swa.py @@ -217,7 +217,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class GraniteSWAMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/helium/modeling_helium.py b/src/transformers/models/helium/modeling_helium.py index a46cf7b926d9..548454e31945 100644 --- a/src/transformers/models/helium/modeling_helium.py +++ b/src/transformers/models/helium/modeling_helium.py @@ -119,7 +119,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HeliumMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py b/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py index 04b717ded41c..7a4077e12c44 100644 --- a/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py +++ b/src/transformers/models/higgs_audio_v2/modeling_higgs_audio_v2.py @@ -45,7 +45,7 @@ logger = logging.get_logger(__name__) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HiggsAudioV2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/hrm_text/modeling_hrm_text.py b/src/transformers/models/hrm_text/modeling_hrm_text.py index d9b145215509..8324d6018202 100644 --- a/src/transformers/models/hrm_text/modeling_hrm_text.py +++ b/src/transformers/models/hrm_text/modeling_hrm_text.py @@ -67,7 +67,7 @@ def extra_repr(self): return f"eps={self.eps}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HrmTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py index 9777ff9a5de2..354e6151b850 100644 --- a/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py +++ b/src/transformers/models/hunyuan_v1_dense/modeling_hunyuan_v1_dense.py @@ -64,7 +64,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HunYuanDenseV1MLP(nn.Module): def __init__(self, config: HunYuanDenseV1Config, layer_idx=None, is_shared_mlp=False): super().__init__() diff --git a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py index 80b4c2e6bf51..6e2b72a629f3 100644 --- a/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py +++ b/src/transformers/models/hunyuan_v1_moe/modeling_hunyuan_v1_moe.py @@ -63,7 +63,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HunYuanMoEV1MLP(nn.Module): def __init__(self, config: HunYuanMoEV1Config): super().__init__() diff --git a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py index 17f9a84a950d..3d49df932cbe 100644 --- a/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py +++ b/src/transformers/models/hunyuan_vl/modeling_hunyuan_vl.py @@ -577,7 +577,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HunYuanVLMLP(nn.Module): def __init__(self, config: HunYuanVLConfig, layer_idx=None, is_shared_mlp=False): super().__init__() diff --git a/src/transformers/models/hy_v3/modeling_hy_v3.py b/src/transformers/models/hy_v3/modeling_hy_v3.py index 476fb0a40ee0..8381b9b78a30 100644 --- a/src/transformers/models/hy_v3/modeling_hy_v3.py +++ b/src/transformers/models/hy_v3/modeling_hy_v3.py @@ -120,7 +120,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HYV3MLP(nn.Module): def __init__(self, config: HYV3Config, intermediate_size: int | None = None): super().__init__() diff --git a/src/transformers/models/hyperclovax/modeling_hyperclovax.py b/src/transformers/models/hyperclovax/modeling_hyperclovax.py index 6401746782e4..a4127440ad3a 100644 --- a/src/transformers/models/hyperclovax/modeling_hyperclovax.py +++ b/src/transformers/models/hyperclovax/modeling_hyperclovax.py @@ -258,7 +258,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class HyperCLOVAXMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/jamba/modeling_jamba.py b/src/transformers/models/jamba/modeling_jamba.py index bfd9f76708ec..50407f7fc03d 100755 --- a/src/transformers/models/jamba/modeling_jamba.py +++ b/src/transformers/models/jamba/modeling_jamba.py @@ -611,7 +611,7 @@ def forward( return contextualized_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class JambaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/laguna/modeling_laguna.py b/src/transformers/models/laguna/modeling_laguna.py index 1eae703c255c..7f7271769172 100644 --- a/src/transformers/models/laguna/modeling_laguna.py +++ b/src/transformers/models/laguna/modeling_laguna.py @@ -138,7 +138,7 @@ def forward(self, x, position_ids, layer_type): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class LagunaMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/llama/modeling_llama.py b/src/transformers/models/llama/modeling_llama.py index 13b7c1370046..b8f1e1cfd737 100644 --- a/src/transformers/models/llama/modeling_llama.py +++ b/src/transformers/models/llama/modeling_llama.py @@ -160,7 +160,7 @@ def apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1): return q_embed, k_embed -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class LlamaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/longcat_flash/modeling_longcat_flash.py b/src/transformers/models/longcat_flash/modeling_longcat_flash.py index a3125d32d15d..61e37db03a2c 100644 --- a/src/transformers/models/longcat_flash/modeling_longcat_flash.py +++ b/src/transformers/models/longcat_flash/modeling_longcat_flash.py @@ -124,7 +124,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class LongcatFlashMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/mellum/modeling_mellum.py b/src/transformers/models/mellum/modeling_mellum.py index 6f92dff8496a..7cb39219068c 100644 --- a/src/transformers/models/mellum/modeling_mellum.py +++ b/src/transformers/models/mellum/modeling_mellum.py @@ -355,7 +355,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return final_hidden_states.reshape(batch_size, sequence_length, hidden_dim) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class MellumMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py b/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py index 88ebdd38ac45..fc93085b3cd1 100644 --- a/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py +++ b/src/transformers/models/mimo_v2_flash/modeling_mimo_v2_flash.py @@ -237,7 +237,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.experts(hidden_states, topk_indices, topk_weights).view(*orig_shape) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class MiMoV2FlashMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/minicpm3/modeling_minicpm3.py b/src/transformers/models/minicpm3/modeling_minicpm3.py index 6249affd5f43..fdd7738861e0 100644 --- a/src/transformers/models/minicpm3/modeling_minicpm3.py +++ b/src/transformers/models/minicpm3/modeling_minicpm3.py @@ -134,7 +134,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class MiniCPM3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/ministral/modeling_ministral.py b/src/transformers/models/ministral/modeling_ministral.py index 423a08d45cda..06e69c77f4d0 100644 --- a/src/transformers/models/ministral/modeling_ministral.py +++ b/src/transformers/models/ministral/modeling_ministral.py @@ -47,7 +47,7 @@ from .configuration_ministral import MinistralConfig -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class MinistralMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/ministral3/modeling_ministral3.py b/src/transformers/models/ministral3/modeling_ministral3.py index d5fa34c8787f..ea585954a761 100644 --- a/src/transformers/models/ministral3/modeling_ministral3.py +++ b/src/transformers/models/ministral3/modeling_ministral3.py @@ -173,7 +173,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Ministral3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/mistral/modeling_mistral.py b/src/transformers/models/mistral/modeling_mistral.py index 19231ec11f2f..d9f2acf01e8b 100644 --- a/src/transformers/models/mistral/modeling_mistral.py +++ b/src/transformers/models/mistral/modeling_mistral.py @@ -32,7 +32,7 @@ from .configuration_mistral import MistralConfig -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class MistralMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/mistral4/modeling_mistral4.py b/src/transformers/models/mistral4/modeling_mistral4.py index e969927f1db4..1f7e6e187af1 100644 --- a/src/transformers/models/mistral4/modeling_mistral4.py +++ b/src/transformers/models/mistral4/modeling_mistral4.py @@ -125,7 +125,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Mistral4MLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py index 6de4add1072a..26667ab38187 100644 --- a/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py +++ b/src/transformers/models/moonshine_streaming/modeling_moonshine_streaming.py @@ -432,7 +432,7 @@ def forward( return MoonshineStreamingEncoderModelOutput(last_hidden_state=hidden_states, attention_mask=attention_mask) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class MoonshinMoonshineStreamingDecoderMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py b/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py index 78cb45f208c2..922a42dfbf7a 100644 --- a/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py +++ b/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py @@ -140,7 +140,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class MuseGlimmerTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py b/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py index da018b1379f0..0c58948558a4 100644 --- a/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py +++ b/src/transformers/models/muse_glimmer_assistant/modeling_muse_glimmer_assistant.py @@ -217,7 +217,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class MuseGlimmerAssistantMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/nomic_bert/modeling_nomic_bert.py b/src/transformers/models/nomic_bert/modeling_nomic_bert.py index a4d7078f8823..50d5b5179a5f 100644 --- a/src/transformers/models/nomic_bert/modeling_nomic_bert.py +++ b/src/transformers/models/nomic_bert/modeling_nomic_bert.py @@ -256,7 +256,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class NomicBertMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/nomic_bert/modular_nomic_bert.py b/src/transformers/models/nomic_bert/modular_nomic_bert.py index 17f88c9d94fd..6d5656ec870e 100644 --- a/src/transformers/models/nomic_bert/modular_nomic_bert.py +++ b/src/transformers/models/nomic_bert/modular_nomic_bert.py @@ -118,7 +118,7 @@ def __init__(self, config): self.o_proj = nn.Linear(config.num_attention_heads * self.head_dim, config.hidden_size, bias=False) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class NomicBertMLP(GemmaMLP): pass diff --git a/src/transformers/models/olmo/modeling_olmo.py b/src/transformers/models/olmo/modeling_olmo.py index fe80867c1468..97f6318f5c77 100644 --- a/src/transformers/models/olmo/modeling_olmo.py +++ b/src/transformers/models/olmo/modeling_olmo.py @@ -60,7 +60,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: ) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class OlmoMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmo2/modeling_olmo2.py b/src/transformers/models/olmo2/modeling_olmo2.py index 0ebc12da6265..38f633866390 100644 --- a/src/transformers/models/olmo2/modeling_olmo2.py +++ b/src/transformers/models/olmo2/modeling_olmo2.py @@ -266,7 +266,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Olmo2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmo3/modeling_olmo3.py b/src/transformers/models/olmo3/modeling_olmo3.py index 4371d333f7df..88f50d8dfb76 100644 --- a/src/transformers/models/olmo3/modeling_olmo3.py +++ b/src/transformers/models/olmo3/modeling_olmo3.py @@ -210,7 +210,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Olmo3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py b/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py index d78c9c4cc5f7..d380d072e4f9 100644 --- a/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py +++ b/src/transformers/models/olmo_hybrid/modeling_olmo_hybrid.py @@ -752,7 +752,7 @@ def forward( return output -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class OlmoHybridMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmoe/modeling_olmoe.py b/src/transformers/models/olmoe/modeling_olmoe.py index aeaf9b7b9886..ce66b0ec68c4 100644 --- a/src/transformers/models/olmoe/modeling_olmoe.py +++ b/src/transformers/models/olmoe/modeling_olmoe.py @@ -118,7 +118,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class OlmoeMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/olmoe/modular_olmoe.py b/src/transformers/models/olmoe/modular_olmoe.py index ee0ed8a63c1d..b78f6f6d425f 100644 --- a/src/transformers/models/olmoe/modular_olmoe.py +++ b/src/transformers/models/olmoe/modular_olmoe.py @@ -52,7 +52,7 @@ class OlmoeRotaryEmbedding(LlamaRotaryEmbedding): pass -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class OlmoeMLP(GemmaMLP): pass diff --git a/src/transformers/models/ovis2/modeling_ovis2.py b/src/transformers/models/ovis2/modeling_ovis2.py index cdc2d544f032..798d661ff098 100644 --- a/src/transformers/models/ovis2/modeling_ovis2.py +++ b/src/transformers/models/ovis2/modeling_ovis2.py @@ -124,7 +124,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Ovis2VisionMLP(nn.Module): def __init__(self, config): super().__init__() @@ -255,7 +255,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Ovis2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py index 28de467e6876..8897fbff0815 100644 --- a/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py +++ b/src/transformers/models/paddleocr_vl/modeling_paddleocr_vl.py @@ -174,7 +174,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class PaddleOCRMLP(nn.Module): def __init__(self, config: PaddleOCRTextConfig): super().__init__() diff --git a/src/transformers/models/pe_audio/modeling_pe_audio.py b/src/transformers/models/pe_audio/modeling_pe_audio.py index 4456d163b263..2750f4bb7043 100644 --- a/src/transformers/models/pe_audio/modeling_pe_audio.py +++ b/src/transformers/models/pe_audio/modeling_pe_audio.py @@ -431,7 +431,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class PeAudioEncoderMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py index 072efaae2b8e..236e08c741f9 100644 --- a/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py +++ b/src/transformers/models/pe_audio_video/modeling_pe_audio_video.py @@ -368,7 +368,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class PeAudioVideoEncoderMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/pe_video/modeling_pe_video.py b/src/transformers/models/pe_video/modeling_pe_video.py index 7c463a62ecba..e5134e0cb223 100644 --- a/src/transformers/models/pe_video/modeling_pe_video.py +++ b/src/transformers/models/pe_video/modeling_pe_video.py @@ -355,7 +355,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class PeVideoEncoderMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen2/modeling_qwen2.py b/src/transformers/models/qwen2/modeling_qwen2.py index 422b00a020f2..4ad77081a407 100644 --- a/src/transformers/models/qwen2/modeling_qwen2.py +++ b/src/transformers/models/qwen2/modeling_qwen2.py @@ -32,7 +32,7 @@ from .configuration_qwen2 import Qwen2Config -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen2MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py index 9940160a4e1f..4f3527f3daf9 100644 --- a/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modeling_qwen2_moe.py @@ -127,7 +127,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen2MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen2_moe/modular_qwen2_moe.py b/src/transformers/models/qwen2_moe/modular_qwen2_moe.py index 4682c82549df..72b52747e097 100644 --- a/src/transformers/models/qwen2_moe/modular_qwen2_moe.py +++ b/src/transformers/models/qwen2_moe/modular_qwen2_moe.py @@ -57,7 +57,7 @@ class Qwen2MoeRotaryEmbedding(Gemma2RotaryEmbedding): pass -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen2MoeMLP(GemmaMLP): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen3/modeling_qwen3.py b/src/transformers/models/qwen3/modeling_qwen3.py index 1d7d0d237eb9..a1aab54c85dc 100644 --- a/src/transformers/models/qwen3/modeling_qwen3.py +++ b/src/transformers/models/qwen3/modeling_qwen3.py @@ -67,7 +67,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen3/modular_qwen3.py b/src/transformers/models/qwen3/modular_qwen3.py index 5082bd817d58..12164c6e5793 100644 --- a/src/transformers/models/qwen3/modular_qwen3.py +++ b/src/transformers/models/qwen3/modular_qwen3.py @@ -50,7 +50,7 @@ class Qwen3RMSNorm(Qwen2RMSNorm): pass -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3MLP(GemmaMLP): pass diff --git a/src/transformers/models/qwen3_5/modeling_qwen3_5.py b/src/transformers/models/qwen3_5/modeling_qwen3_5.py index 42ff67e21f84..a5ed4f96c0eb 100644 --- a/src/transformers/models/qwen3_5/modeling_qwen3_5.py +++ b/src/transformers/models/qwen3_5/modeling_qwen3_5.py @@ -777,7 +777,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3_5MLP(nn.Module): def __init__(self, config: Qwen3_5Config, intermediate_size: int): super().__init__() diff --git a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py index 25ed3c4c0e41..33a9bdbbe0b6 100644 --- a/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py +++ b/src/transformers/models/qwen3_5_moe/modeling_qwen3_5_moe.py @@ -777,7 +777,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3_5MoeMLP(nn.Module): def __init__(self, config: Qwen3_5MoeConfig, intermediate_size: int): super().__init__() diff --git a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py index 0f706aad1257..0a2181931c6a 100644 --- a/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py +++ b/src/transformers/models/qwen3_moe/modeling_qwen3_moe.py @@ -190,7 +190,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3MoeMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen3_next/modeling_qwen3_next.py b/src/transformers/models/qwen3_next/modeling_qwen3_next.py index 844bfb0a4400..ea86bc03d8a1 100644 --- a/src/transformers/models/qwen3_next/modeling_qwen3_next.py +++ b/src/transformers/models/qwen3_next/modeling_qwen3_next.py @@ -767,7 +767,7 @@ def forward( return output -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3NextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py index e9df9a510119..c64e2edb8ee7 100644 --- a/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py +++ b/src/transformers/models/qwen3_omni_moe/modeling_qwen3_omni_moe.py @@ -1473,7 +1473,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3OmniMoeThinkerTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() @@ -2377,7 +2377,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3OmniMoeMLP(nn.Module): def __init__(self, config): super().__init__() @@ -2689,7 +2689,7 @@ class Qwen3OmniMoeTalkerRotaryEmbedding(Qwen3OmniMoeThinkerTextRotaryEmbedding): pass -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3OmniMoeTalkerTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() @@ -3394,7 +3394,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3OmniMoeCode2WavMlp(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py index 90f9e554cc9e..2da4fa4dee35 100644 --- a/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py +++ b/src/transformers/models/qwen3_vl/modeling_qwen3_vl.py @@ -506,7 +506,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3VLTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py index 4f75cfa7bb79..299594dd2e74 100644 --- a/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py +++ b/src/transformers/models/qwen3_vl_moe/modeling_qwen3_vl_moe.py @@ -302,7 +302,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen3VLMoeTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py b/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py index b07bede3e130..9935fd896fd6 100644 --- a/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py +++ b/src/transformers/models/qwen4_exp/modeling_qwen4_exp.py @@ -912,7 +912,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class Qwen4ExpTextMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/smollm3/modeling_smollm3.py b/src/transformers/models/smollm3/modeling_smollm3.py index eafa66416bdb..9a381adaca49 100644 --- a/src/transformers/models/smollm3/modeling_smollm3.py +++ b/src/transformers/models/smollm3/modeling_smollm3.py @@ -268,7 +268,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class SmolLM3MLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/solar_open/modeling_solar_open.py b/src/transformers/models/solar_open/modeling_solar_open.py index d4c40a1227d3..f871cab9bea4 100644 --- a/src/transformers/models/solar_open/modeling_solar_open.py +++ b/src/transformers/models/solar_open/modeling_solar_open.py @@ -83,7 +83,7 @@ def forward( return hidden_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class SolarOpenMLP(nn.Module): def __init__(self, config, intermediate_size=None): super().__init__() diff --git a/src/transformers/models/vaultgemma/modeling_vaultgemma.py b/src/transformers/models/vaultgemma/modeling_vaultgemma.py index 13ae2b5e841a..ba70ef4cc6e2 100644 --- a/src/transformers/models/vaultgemma/modeling_vaultgemma.py +++ b/src/transformers/models/vaultgemma/modeling_vaultgemma.py @@ -63,7 +63,7 @@ def extra_repr(self): return f"{tuple(self.weight.shape)}, eps={self.eps}" -@use_kernel_forward_from_hub("GeGLUMLP") +@use_kernel_forward_from_hub("GeGLUMLP", condition=lambda module: module.config.hidden_act == "gelu_pytorch_tanh") class VaultGemmaMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/vibevoice/modeling_vibevoice.py b/src/transformers/models/vibevoice/modeling_vibevoice.py index 0e068bff70e6..16d7b8a17fd3 100644 --- a/src/transformers/models/vibevoice/modeling_vibevoice.py +++ b/src/transformers/models/vibevoice/modeling_vibevoice.py @@ -118,7 +118,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class VibeVoiceMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py index 5d62eb163201..152eff8257a7 100644 --- a/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py +++ b/src/transformers/models/voxtral_realtime/modeling_voxtral_realtime.py @@ -396,7 +396,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class VoxtralRealtimeMLP(nn.Module): def __init__(self, config): super().__init__() @@ -682,7 +682,7 @@ def forward( return attn_output, attn_weights -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class VoxtralRealtimeTextMLP(nn.Module): def __init__(self, config): super().__init__() diff --git a/src/transformers/models/youtu/modeling_youtu.py b/src/transformers/models/youtu/modeling_youtu.py index 96ccbd9f7fff..4c7ca03d1f6e 100644 --- a/src/transformers/models/youtu/modeling_youtu.py +++ b/src/transformers/models/youtu/modeling_youtu.py @@ -127,7 +127,7 @@ def forward(self, x, position_ids): return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) -@use_kernel_forward_from_hub("SwiGLUMLP") +@use_kernel_forward_from_hub("SwiGLUMLP", condition=lambda module: module.config.hidden_act == "silu") class YoutuMLP(nn.Module): def __init__(self, config): super().__init__() From f00c2f16cccafdaa5bd577f2fda17d6fad82fd57 Mon Sep 17 00:00:00 2001 From: vasqu Date: Mon, 31 Aug 2026 19:47:32 +0000 Subject: [PATCH 8/8] fix with attr map --- .../models/diffusion_gemma/configuration_diffusion_gemma.py | 1 + .../models/diffusion_gemma/modeling_diffusion_gemma.py | 2 +- .../models/diffusion_gemma/modular_diffusion_gemma.py | 2 +- src/transformers/models/gemma2/configuration_gemma2.py | 1 + src/transformers/models/gemma2/modeling_gemma2.py | 2 +- src/transformers/models/gemma2/modular_gemma2.py | 6 ++---- src/transformers/models/gemma3/configuration_gemma3.py | 1 + src/transformers/models/gemma3/modeling_gemma3.py | 2 +- src/transformers/models/gemma3n/configuration_gemma3n.py | 1 + src/transformers/models/gemma3n/modeling_gemma3n.py | 2 +- src/transformers/models/gemma4/configuration_gemma4.py | 2 ++ src/transformers/models/gemma4/modeling_gemma4.py | 4 ++-- .../models/gemma4_unified/configuration_gemma4_unified.py | 1 + .../models/gemma4_unified/modeling_gemma4_unified.py | 2 +- .../models/muse_glimmer/configuration_muse_glimmer.py | 1 + .../models/muse_glimmer/modeling_muse_glimmer.py | 2 +- src/transformers/models/t5gemma/configuration_t5gemma.py | 1 + src/transformers/models/t5gemma/modeling_t5gemma.py | 2 +- src/transformers/models/t5gemma2/configuration_t5gemma2.py | 2 ++ src/transformers/models/t5gemma2/modeling_t5gemma2.py | 2 +- .../models/vaultgemma/configuration_vaultgemma.py | 1 + src/transformers/models/vaultgemma/modeling_vaultgemma.py | 2 +- 22 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/diffusion_gemma/configuration_diffusion_gemma.py b/src/transformers/models/diffusion_gemma/configuration_diffusion_gemma.py index cc096cea1ca3..b71cc6876abb 100644 --- a/src/transformers/models/diffusion_gemma/configuration_diffusion_gemma.py +++ b/src/transformers/models/diffusion_gemma/configuration_diffusion_gemma.py @@ -75,6 +75,7 @@ class DiffusionGemmaTextConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 262_144 hidden_size: int = 2304 diff --git a/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py b/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py index 02c205b63909..e8d8d477d8c1 100644 --- a/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py +++ b/src/transformers/models/diffusion_gemma/modeling_diffusion_gemma.py @@ -507,7 +507,7 @@ def __init__(self, config: DiffusionGemmaTextConfig, layer_idx: int): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) diff --git a/src/transformers/models/diffusion_gemma/modular_diffusion_gemma.py b/src/transformers/models/diffusion_gemma/modular_diffusion_gemma.py index b0b864d5c93f..00d8794a84a9 100644 --- a/src/transformers/models/diffusion_gemma/modular_diffusion_gemma.py +++ b/src/transformers/models/diffusion_gemma/modular_diffusion_gemma.py @@ -464,7 +464,7 @@ def __init__(self, config: DiffusionGemmaTextConfig, layer_idx: int): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] class DiffusionGemmaTextRouter(Gemma4TextRouter): diff --git a/src/transformers/models/gemma2/configuration_gemma2.py b/src/transformers/models/gemma2/configuration_gemma2.py index 11d7b012099f..a04dcbb14261 100644 --- a/src/transformers/models/gemma2/configuration_gemma2.py +++ b/src/transformers/models/gemma2/configuration_gemma2.py @@ -64,6 +64,7 @@ class Gemma2Config(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 256000 hidden_size: int = 2304 diff --git a/src/transformers/models/gemma2/modeling_gemma2.py b/src/transformers/models/gemma2/modeling_gemma2.py index 5e53a363628d..5a295e03442e 100644 --- a/src/transformers/models/gemma2/modeling_gemma2.py +++ b/src/transformers/models/gemma2/modeling_gemma2.py @@ -76,7 +76,7 @@ def __init__(self, config): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) diff --git a/src/transformers/models/gemma2/modular_gemma2.py b/src/transformers/models/gemma2/modular_gemma2.py index d798196f1807..3d01ad1e49e2 100644 --- a/src/transformers/models/gemma2/modular_gemma2.py +++ b/src/transformers/models/gemma2/modular_gemma2.py @@ -18,7 +18,6 @@ import torch.nn as nn from huggingface_hub.dataclasses import strict -from ...activations import ACT2FN from ...cache_utils import Cache, DynamicCache from ...configuration_utils import PreTrainedConfig from ...masking_utils import create_causal_mask, create_sliding_window_causal_mask @@ -92,6 +91,7 @@ class Gemma2Config(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 256000 hidden_size: int = 2304 @@ -141,9 +141,7 @@ class Gemma2RMSNorm(GemmaRMSNorm): class Gemma2MLP(GemmaMLP): - def __init__(self, config): - super().__init__(config) - self.act_fn = ACT2FN[config.hidden_activation] + pass class Gemma2RotaryEmbedding(GemmaRotaryEmbedding): diff --git a/src/transformers/models/gemma3/configuration_gemma3.py b/src/transformers/models/gemma3/configuration_gemma3.py index b640a906fbef..9a40687acc83 100644 --- a/src/transformers/models/gemma3/configuration_gemma3.py +++ b/src/transformers/models/gemma3/configuration_gemma3.py @@ -73,6 +73,7 @@ class Gemma3TextConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 262_208 hidden_size: int = 2304 diff --git a/src/transformers/models/gemma3/modeling_gemma3.py b/src/transformers/models/gemma3/modeling_gemma3.py index c091cf5ccc76..cc43497588c8 100644 --- a/src/transformers/models/gemma3/modeling_gemma3.py +++ b/src/transformers/models/gemma3/modeling_gemma3.py @@ -127,7 +127,7 @@ def __init__(self, config: Gemma3TextConfig): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) diff --git a/src/transformers/models/gemma3n/configuration_gemma3n.py b/src/transformers/models/gemma3n/configuration_gemma3n.py index 8f197fa3bb3f..462c3a962502 100644 --- a/src/transformers/models/gemma3n/configuration_gemma3n.py +++ b/src/transformers/models/gemma3n/configuration_gemma3n.py @@ -94,6 +94,7 @@ class Gemma3nTextConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 262_400 hidden_size: int = 2048 diff --git a/src/transformers/models/gemma3n/modeling_gemma3n.py b/src/transformers/models/gemma3n/modeling_gemma3n.py index 57dfb6b2d4e4..837ca6b29497 100644 --- a/src/transformers/models/gemma3n/modeling_gemma3n.py +++ b/src/transformers/models/gemma3n/modeling_gemma3n.py @@ -957,7 +957,7 @@ def __init__(self, config: Gemma3nTextConfig, layer_idx: int = 0): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] self.activation_sparsity = config.activation_sparsity_pattern[layer_idx] def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: diff --git a/src/transformers/models/gemma4/configuration_gemma4.py b/src/transformers/models/gemma4/configuration_gemma4.py index ff970b4c7ef5..c381eab0dc28 100644 --- a/src/transformers/models/gemma4/configuration_gemma4.py +++ b/src/transformers/models/gemma4/configuration_gemma4.py @@ -148,6 +148,7 @@ class Gemma4TextConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 262_144 hidden_size: int = 2304 @@ -266,6 +267,7 @@ class Gemma4VisionConfig(PreTrainedConfig): "encoder.layers.*.mlp.up_proj": "colwise", "encoder.layers.*.mlp.down_proj": "rowwise", } + attribute_map = {"hidden_act": "hidden_activation"} default_theta = 100.0 diff --git a/src/transformers/models/gemma4/modeling_gemma4.py b/src/transformers/models/gemma4/modeling_gemma4.py index c2ea67780236..7ae059bac03a 100644 --- a/src/transformers/models/gemma4/modeling_gemma4.py +++ b/src/transformers/models/gemma4/modeling_gemma4.py @@ -698,7 +698,7 @@ def __init__(self, config: Gemma4VisionConfig): self.gate_proj = Gemma4ClippableLinear(config, self.hidden_size, self.intermediate_size) self.up_proj = Gemma4ClippableLinear(config, self.hidden_size, self.intermediate_size) self.down_proj = Gemma4ClippableLinear(config, self.intermediate_size, self.hidden_size) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) @@ -1076,7 +1076,7 @@ def __init__(self, config: Gemma4TextConfig, layer_idx: int): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) diff --git a/src/transformers/models/gemma4_unified/configuration_gemma4_unified.py b/src/transformers/models/gemma4_unified/configuration_gemma4_unified.py index 0b93b9a1701d..70968133126e 100644 --- a/src/transformers/models/gemma4_unified/configuration_gemma4_unified.py +++ b/src/transformers/models/gemma4_unified/configuration_gemma4_unified.py @@ -120,6 +120,7 @@ class Gemma4UnifiedTextConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 262_144 hidden_size: int = 2304 diff --git a/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py b/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py index b9569afeb55d..896ab6cf6370 100644 --- a/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py +++ b/src/transformers/models/gemma4_unified/modeling_gemma4_unified.py @@ -469,7 +469,7 @@ def __init__(self, config: Gemma4UnifiedTextConfig, layer_idx: int): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) diff --git a/src/transformers/models/muse_glimmer/configuration_muse_glimmer.py b/src/transformers/models/muse_glimmer/configuration_muse_glimmer.py index 19cb5a57eb43..af3d52ab5ad3 100644 --- a/src/transformers/models/muse_glimmer/configuration_muse_glimmer.py +++ b/src/transformers/models/muse_glimmer/configuration_muse_glimmer.py @@ -105,6 +105,7 @@ class MuseGlimmerTextConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 202_048 hidden_size: int = 6656 diff --git a/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py b/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py index 922a42dfbf7a..095f9475f7b2 100644 --- a/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py +++ b/src/transformers/models/muse_glimmer/modeling_muse_glimmer.py @@ -150,7 +150,7 @@ def __init__(self, config): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) diff --git a/src/transformers/models/t5gemma/configuration_t5gemma.py b/src/transformers/models/t5gemma/configuration_t5gemma.py index 9de40c832259..245d619c08ba 100644 --- a/src/transformers/models/t5gemma/configuration_t5gemma.py +++ b/src/transformers/models/t5gemma/configuration_t5gemma.py @@ -64,6 +64,7 @@ class T5GemmaModuleConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 256000 hidden_size: int = 2304 diff --git a/src/transformers/models/t5gemma/modeling_t5gemma.py b/src/transformers/models/t5gemma/modeling_t5gemma.py index ddbacedb7782..e1b5c231a2d6 100644 --- a/src/transformers/models/t5gemma/modeling_t5gemma.py +++ b/src/transformers/models/t5gemma/modeling_t5gemma.py @@ -87,7 +87,7 @@ def __init__(self, config): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] self.dropout = nn.Dropout(config.dropout_rate) def forward(self, x): diff --git a/src/transformers/models/t5gemma2/configuration_t5gemma2.py b/src/transformers/models/t5gemma2/configuration_t5gemma2.py index ebddcb7810f2..fdf08a82317b 100644 --- a/src/transformers/models/t5gemma2/configuration_t5gemma2.py +++ b/src/transformers/models/t5gemma2/configuration_t5gemma2.py @@ -60,6 +60,7 @@ class T5Gemma2TextConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 262_208 hidden_size: int = 2304 @@ -244,6 +245,7 @@ class T5Gemma2DecoderConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 262_208 hidden_size: int = 2304 diff --git a/src/transformers/models/t5gemma2/modeling_t5gemma2.py b/src/transformers/models/t5gemma2/modeling_t5gemma2.py index d54e95d76a59..6164c3b7e22d 100644 --- a/src/transformers/models/t5gemma2/modeling_t5gemma2.py +++ b/src/transformers/models/t5gemma2/modeling_t5gemma2.py @@ -81,7 +81,7 @@ def __init__(self, config: T5Gemma2TextConfig): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] self.dropout = nn.Dropout(config.dropout_rate) def forward(self, x): diff --git a/src/transformers/models/vaultgemma/configuration_vaultgemma.py b/src/transformers/models/vaultgemma/configuration_vaultgemma.py index a60b7e8edc0c..8c646db44a0b 100644 --- a/src/transformers/models/vaultgemma/configuration_vaultgemma.py +++ b/src/transformers/models/vaultgemma/configuration_vaultgemma.py @@ -63,6 +63,7 @@ class VaultGemmaConfig(PreTrainedConfig): "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } + attribute_map = {"hidden_act": "hidden_activation"} vocab_size: int = 256000 hidden_size: int = 2304 diff --git a/src/transformers/models/vaultgemma/modeling_vaultgemma.py b/src/transformers/models/vaultgemma/modeling_vaultgemma.py index ba70ef4cc6e2..4eb81e8632a0 100644 --- a/src/transformers/models/vaultgemma/modeling_vaultgemma.py +++ b/src/transformers/models/vaultgemma/modeling_vaultgemma.py @@ -73,7 +73,7 @@ def __init__(self, config): self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) - self.act_fn = ACT2FN[config.hidden_activation] + self.act_fn = ACT2FN[config.hidden_act] def forward(self, x): down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))