{"architectures":["Qwen3MoeForCausalLM"],"attention_bias":false,"attention_dropout":0,"bos_token_id":151643,"decoder_sparse_step":1,"eos_token_id":151645,"head_dim":128,"hidden_act":"silu","hidden_size":2048,"initializer_range":0.02,"intermediate_size":6144,"max_position_embeddings":262144,"max_window_layers":48,"mlp_only_layers":[],"model_type":"qwen3_moe","moe_intermediate_size":768,"norm_topk_prob":true,"num_attention_heads":32,"num_experts":128,"num_experts_per_tok":8,"num_hidden_layers":48,"num_key_value_heads":4,"output_router_logits":false,"rms_norm_eps":0.000001,"rope_scaling":null,"rope_theta":10000000,"router_aux_loss_coef":0.001,"sliding_window":null,"tie_word_embeddings":false,"torch_dtype":"bfloat16","transformers_version":"4.51.0","use_cache":true,"use_sliding_window":false,"vocab_size":151936}