diff --git a/mergekit/_data/architectures/qwen3_5_moe.json b/mergekit/_data/architectures/qwen3_5_moe.json new file mode 100644 index 00000000..3a3c7d96 --- /dev/null +++ b/mergekit/_data/architectures/qwen3_5_moe.json @@ -0,0 +1,229 @@ +{ + "kind": "modular", + "model_type": "qwen3_5_moe", + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "vocab_size_config_key": "text_config.vocab_size", + "tagalong_files": [ + "preprocessor_config.json", + "video_preprocessor_config.json", + "chat_template.jinja", + "merges.txt", + "tokenizer.json", + "tokenizer_config.json" + ], + "modules": { + "default": { + "architecture": { + "architectures": [ + "Qwen3ForCausalLM" + ], + "model_type": "qwen3_5_moe_text", + "num_layers_config_key": "text_config.num_hidden_layers", + "pre_weights": [ + { + "name": "model.language_model.embed_tokens.weight", + "is_embed": true + } + ], + "post_weights": [ + { + "name": "model.language_model.norm.weight" + }, + { + "name": "lm_head.weight", + "is_embed": true, + "optional": true, + "tied_names": [ + "model.language_model.embed_tokens.weight" + ] + } + ], + "layer_templates": { + "weights": [ + { + "name": "model.language_model.layers.${layer_index}.input_layernorm.weight" + }, + { + "name": "model.language_model.layers.${layer_index}.post_attention_layernorm.weight" + }, + { + "name": "model.language_model.layers.${layer_index}.self_attn.k_norm.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.self_attn.k_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.self_attn.o_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.self_attn.q_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.self_attn.v_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.self_attn.q_norm.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.mlp.gate.weight" + }, + { + "name": "model.language_model.layers.${layer_index}.mlp.experts.gate_up_proj" + }, + { + "name": "model.language_model.layers.${layer_index}.mlp.experts.down_proj" + }, + { + "name": "model.language_model.layers.${layer_index}.mlp.shared_expert.gate_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.mlp.shared_expert.up_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.mlp.shared_expert.down_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.mlp.shared_expert_gate.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.in_proj_qkv.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.in_proj_a.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.in_proj_b.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.in_proj_z.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.out_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.norm.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.A_log", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.D", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.dt_bias", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.conv1d.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.conv1d.bias", + "optional": true + } + ] + } + } + }, + "visual": { + "architecture": { + "model_type": "", + "architectures": [], + "num_layers_config_key": "vision_config.depth", + "pre_weights": [ + { + "name": "model.visual.patch_embed.proj.weight", + "is_embed": true + }, + { + "name": "model.visual.patch_embed.proj.bias" + }, + { + "name": "model.visual.pos_embed.weight", + "is_embed": true + } + ], + "post_weights": [ + { + "name": "model.visual.merger.norm.weight" + }, + { + "name": "model.visual.merger.norm.bias" + }, + { + "name": "model.visual.merger.linear_fc1.weight" + }, + { + "name": "model.visual.merger.linear_fc1.bias" + }, + { + "name": "model.visual.merger.linear_fc2.weight" + }, + { + "name": "model.visual.merger.linear_fc2.bias" + } + ], + "layer_templates": { + "weights": [ + { + "name": "model.visual.blocks.${layer_index}.norm1.weight" + }, + { + "name": "model.visual.blocks.${layer_index}.norm1.bias" + }, + { + "name": "model.visual.blocks.${layer_index}.norm2.weight" + }, + { + "name": "model.visual.blocks.${layer_index}.norm2.bias" + }, + { + "name": "model.visual.blocks.${layer_index}.attn.qkv.weight" + }, + { + "name": "model.visual.blocks.${layer_index}.attn.qkv.bias" + }, + { + "name": "model.visual.blocks.${layer_index}.attn.proj.weight" + }, + { + "name": "model.visual.blocks.${layer_index}.attn.proj.bias" + }, + { + "name": "model.visual.blocks.${layer_index}.mlp.linear_fc1.weight" + }, + { + "name": "model.visual.blocks.${layer_index}.mlp.linear_fc1.bias" + }, + { + "name": "model.visual.blocks.${layer_index}.mlp.linear_fc2.weight" + }, + { + "name": "model.visual.blocks.${layer_index}.mlp.linear_fc2.bias" + } + ] + } + } + } + } +} \ No newline at end of file diff --git a/mergekit/_data/architectures/qwen3_vl.json b/mergekit/_data/architectures/qwen3_vl.json index a257dbf9..91d8cfe6 100644 --- a/mergekit/_data/architectures/qwen3_vl.json +++ b/mergekit/_data/architectures/qwen3_vl.json @@ -22,6 +22,18 @@ { "name": "model.language_model.embed_tokens.weight", "is_embed": true + }, + { + "name": "model.language_model.layers.linear_attn.in_proj_qkv.weight", + "optional": true + }, + { + "name": "model.language_model.layers.linear_attn.in_proj_z.weight", + "optional": true + }, + { + "name": "model.language_model.layers.linear_attn.out_proj.weight", + "optional": true } ], "post_weights": [ @@ -71,6 +83,38 @@ }, { "name": "model.language_model.layers.${layer_index}.self_attn.o_proj.weight" + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.in_proj_qkv.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.in_proj_z.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.out_proj.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.dt_bias", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.A_log", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.D", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.conv1d.weight", + "optional": true + }, + { + "name": "model.language_model.layers.${layer_index}.linear_attn.conv1d.bias", + "optional": true } ] }