| --- |
| library_name: transformers |
| pipeline_tag: text-generation |
| inference: true |
| widget: |
| - text: Hello! |
| example_title: Hello world |
| group: Python |
| base_model: |
| - Qwen/Qwen3-Next-80B-A3B-Instruct |
| --- |
| |
| This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct). |
|
|
| ### Example usage: |
|
|
| - vLLM |
|
|
| ```bash |
| VLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \ |
| vllm serve tiny-random/qwen3-next-moe \ |
| --tensor-parallel-size 4 \ |
| --max-model-len 262144 \ |
| --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":2}' |
| |
| ``` |
|
|
| - SGLang |
|
|
| ```bash |
| SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1 \ |
| python -m sglang.launch_server \ |
| --model-path tiny-random/qwen3-next-moe \ |
| --tp-size 4 --context-length 262144 \ |
| --mem-fraction-static 0.8 \ |
| --speculative-algo NEXTN \ |
| --speculative-num-steps 3 \ |
| --speculative-eagle-topk 1 \ |
| --speculative-num-draft-tokens 4 |
| |
| ``` |
|
|
| - Transformers |
|
|
| ```python |
| from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline |
| model_id = "tiny-random/qwen3-next-moe" |
| |
| # load the tokenizer and the model |
| tokenizer = AutoTokenizer.from_pretrained(model_id) |
| model = AutoModelForCausalLM.from_pretrained( |
| model_id, |
| dtype="auto", |
| device_map="cuda", |
| ) |
| # prepare the model input |
| prompt = "Give me a short introduction to large language model." |
| messages = [ |
| {"role": "user", "content": prompt}, |
| ] |
| text = tokenizer.apply_chat_template( |
| messages, |
| tokenize=False, |
| add_generation_prompt=True, |
| ) |
| model_inputs = tokenizer([text], return_tensors="pt").to(model.device) |
| # conduct text completion |
| generated_ids = model.generate( |
| **model_inputs, |
| max_new_tokens=8, |
| ) |
| output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist() |
| content = tokenizer.decode(output_ids, skip_special_tokens=True) |
| print("content:", content) |
| ``` |
|
|
| ### Codes to create this repo: |
|
|
| ```python |
| from copy import deepcopy |
| |
| import torch |
| import torch.nn as nn |
| from transformers import ( |
| AutoConfig, |
| AutoModelForCausalLM, |
| AutoTokenizer, |
| GenerationConfig, |
| pipeline, |
| set_seed, |
| ) |
| |
| source_model_id = "Qwen/Qwen3-Next-80B-A3B-Instruct" |
| save_folder = "/tmp/tiny-random/qwen3-next-moe" |
| |
| tokenizer = AutoTokenizer.from_pretrained( |
| source_model_id, trust_remote_code=True, |
| ) |
| tokenizer.save_pretrained(save_folder) |
| |
| config = AutoConfig.from_pretrained( |
| source_model_id, trust_remote_code=True, |
| ) |
| config._name_or_path = source_model_id |
| config.hidden_size = 8 |
| config.intermediate_size = 32 |
| config.head_dim = 32 |
| config.num_key_value_heads = 8 |
| config.num_attention_heads = 16 |
| config.num_hidden_layers = 4 |
| config.tie_word_embeddings = False |
| config.linear_num_key_heads = 8 |
| config.linear_num_value_heads = 16 |
| config.moe_intermediate_size = 32 |
| config.num_experts = 32 |
| config.num_experts_per_tok = 10 |
| config.layer_types = config.layer_types[:4] |
| config.shared_expert_intermediate_size = 32 |
| model = AutoModelForCausalLM.from_config( |
| config, |
| torch_dtype=torch.bfloat16, |
| trust_remote_code=True, |
| ) |
| model.generation_config = GenerationConfig.from_pretrained( |
| source_model_id, trust_remote_code=True, |
| ) |
| # MTP |
| model.mtp = nn.ModuleDict({ |
| "pre_fc_norm_embedding": nn.RMSNorm(config.hidden_size), |
| "fc": nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False), |
| "norm": nn.RMSNorm(config.hidden_size), |
| "pre_fc_norm_hidden": nn.RMSNorm(config.hidden_size), |
| "layers": nn.ModuleList([deepcopy(model.model.layers[3])]), |
| }) |
| model = model.to(torch.bfloat16) |
| set_seed(42) |
| with torch.no_grad(): |
| for name, p in sorted(model.named_parameters()): |
| torch.nn.init.normal_(p, 0, 0.1) |
| print(name, p.shape) |
| model.save_pretrained(save_folder) |
| ``` |
|
|
| ### Printing the model: |
|
|
| ```text |
| Qwen3NextForCausalLM( |
| (model): Qwen3NextModel( |
| (embed_tokens): Embedding(151936, 8) |
| (layers): ModuleList( |
| (0-2): 3 x Qwen3NextDecoderLayer( |
| (linear_attn): Qwen3NextGatedDeltaNet( |
| (act): SiLU() |
| (conv1d): Conv1d(4096, 4096, kernel_size=(4,), stride=(1,), padding=(3,), groups=4096, bias=False) |
| (in_proj_qkvz): Linear(in_features=8, out_features=6144, bias=False) |
| (in_proj_ba): Linear(in_features=8, out_features=32, bias=False) |
| (norm): FusedRMSNormGated(128, eps=1e-06, activation=silu) |
| (out_proj): Linear(in_features=2048, out_features=8, bias=False) |
| ) |
| (mlp): Qwen3NextSparseMoeBlock( |
| (gate): Linear(in_features=8, out_features=32, bias=False) |
| (experts): ModuleList( |
| (0-31): 32 x Qwen3NextMLP( |
| (gate_proj): Linear(in_features=8, out_features=32, bias=False) |
| (up_proj): Linear(in_features=8, out_features=32, bias=False) |
| (down_proj): Linear(in_features=32, out_features=8, bias=False) |
| (act_fn): SiLU() |
| ) |
| ) |
| (shared_expert): Qwen3NextMLP( |
| (gate_proj): Linear(in_features=8, out_features=32, bias=False) |
| (up_proj): Linear(in_features=8, out_features=32, bias=False) |
| (down_proj): Linear(in_features=32, out_features=8, bias=False) |
| (act_fn): SiLU() |
| ) |
| (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False) |
| ) |
| (input_layernorm): Qwen3NextRMSNorm((8,), eps=1e-06) |
| (post_attention_layernorm): Qwen3NextRMSNorm((8,), eps=1e-06) |
| ) |
| (3): Qwen3NextDecoderLayer( |
| (self_attn): Qwen3NextAttention( |
| (q_proj): Linear(in_features=8, out_features=1024, bias=False) |
| (k_proj): Linear(in_features=8, out_features=256, bias=False) |
| (v_proj): Linear(in_features=8, out_features=256, bias=False) |
| (o_proj): Linear(in_features=512, out_features=8, bias=False) |
| (q_norm): Qwen3NextRMSNorm((32,), eps=1e-06) |
| (k_norm): Qwen3NextRMSNorm((32,), eps=1e-06) |
| ) |
| (mlp): Qwen3NextSparseMoeBlock( |
| (gate): Linear(in_features=8, out_features=32, bias=False) |
| (experts): ModuleList( |
| (0-31): 32 x Qwen3NextMLP( |
| (gate_proj): Linear(in_features=8, out_features=32, bias=False) |
| (up_proj): Linear(in_features=8, out_features=32, bias=False) |
| (down_proj): Linear(in_features=32, out_features=8, bias=False) |
| (act_fn): SiLU() |
| ) |
| ) |
| (shared_expert): Qwen3NextMLP( |
| (gate_proj): Linear(in_features=8, out_features=32, bias=False) |
| (up_proj): Linear(in_features=8, out_features=32, bias=False) |
| (down_proj): Linear(in_features=32, out_features=8, bias=False) |
| (act_fn): SiLU() |
| ) |
| (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False) |
| ) |
| (input_layernorm): Qwen3NextRMSNorm((8,), eps=1e-06) |
| (post_attention_layernorm): Qwen3NextRMSNorm((8,), eps=1e-06) |
| ) |
| ) |
| (norm): Qwen3NextRMSNorm((8,), eps=1e-06) |
| (rotary_emb): Qwen3NextRotaryEmbedding() |
| ) |
| (lm_head): Linear(in_features=8, out_features=151936, bias=False) |
| (mtp): ModuleDict( |
| (pre_fc_norm_embedding): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (fc): Linear(in_features=16, out_features=8, bias=False) |
| (norm): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (pre_fc_norm_hidden): RMSNorm((8,), eps=None, elementwise_affine=True) |
| (layers): ModuleList( |
| (0): Qwen3NextDecoderLayer( |
| (self_attn): Qwen3NextAttention( |
| (q_proj): Linear(in_features=8, out_features=1024, bias=False) |
| (k_proj): Linear(in_features=8, out_features=256, bias=False) |
| (v_proj): Linear(in_features=8, out_features=256, bias=False) |
| (o_proj): Linear(in_features=512, out_features=8, bias=False) |
| (q_norm): Qwen3NextRMSNorm((32,), eps=1e-06) |
| (k_norm): Qwen3NextRMSNorm((32,), eps=1e-06) |
| ) |
| (mlp): Qwen3NextSparseMoeBlock( |
| (gate): Linear(in_features=8, out_features=32, bias=False) |
| (experts): ModuleList( |
| (0-31): 32 x Qwen3NextMLP( |
| (gate_proj): Linear(in_features=8, out_features=32, bias=False) |
| (up_proj): Linear(in_features=8, out_features=32, bias=False) |
| (down_proj): Linear(in_features=32, out_features=8, bias=False) |
| (act_fn): SiLU() |
| ) |
| ) |
| (shared_expert): Qwen3NextMLP( |
| (gate_proj): Linear(in_features=8, out_features=32, bias=False) |
| (up_proj): Linear(in_features=8, out_features=32, bias=False) |
| (down_proj): Linear(in_features=32, out_features=8, bias=False) |
| (act_fn): SiLU() |
| ) |
| (shared_expert_gate): Linear(in_features=8, out_features=1, bias=False) |
| ) |
| (input_layernorm): Qwen3NextRMSNorm((8,), eps=1e-06) |
| (post_attention_layernorm): Qwen3NextRMSNorm((8,), eps=1e-06) |
| ) |
| ) |
| ) |
| ) |
| ``` |
