deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 14, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 12 }, { "name": "blk.0.attn_norm.weight", "offset": 724992000, "shape": [ 5120 ], "size": 294912000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 725012480, "shape": [ 12288, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 768266240, "shape": [ 5120, 12288 ], "size": 43253760, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 803655680, "shape": [ 5120, 12288 ], "size": 35389440, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 839045120, "shape": [ 5120 ], "size": 35389440, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 839065600, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 839067648, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.0.attn_kv_b.weight", "offset": 840726528, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 850163712, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 897349632, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 897355776, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.0.attn_q_b.weight", "offset": 901779456, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 923013120, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 926289920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 937103360, "shape": [ 5120, 3072 ], "size": 10813440, "type": 12 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 945950720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 954798080, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 954800128, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.1.attn_kv_b.weight", "offset": 956459008, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 965896192, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "\u0000\u0015\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.1.attn_q_b.we", "offset": 1013082112, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1013088256, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.1.attn_q_b.weight", "offset": 1017511936, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "output_norm.weight", "offset": 1038745600, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1038766080, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1038786560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1903861760, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 12 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2611650560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 3319439360, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3319459840, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 3322736640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 3333550080, "shape": [ 5120, 3072 ], "size": 10813440, "type": 12 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 3342397440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 3351244800, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 3351246848, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.2.attn_kv_b.weight", "offset": 3352905728, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 3362342912, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 3409528832, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 3409534976, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.2.attn_q_b.weight", "offset": 3413958656, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 3435192320, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3435212800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 4300288000, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 12 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 5008076800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 5715865600, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 5715886080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 5719162880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 5729976320, "shape": [ 5120, 3072 ], "size": 10813440, "type": 12 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 5738823680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 5747671040, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 5747673088, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.3.attn_kv_b.weight", "offset": 5749331968, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 5758769152, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 5805955072, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 5805961216, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.3.attn_q_b.weight", "offset": 5810384896, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 5831618560, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 5831639040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 6696714240, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 12 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 7404503040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 8112291840, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 8112312320, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 8115589120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 8126402560, "shape": [ 5120, 3072 ], "size": 10813440, "type": 12 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 8135249920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 8144097280, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 8144099328, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.4.attn_kv_b.weight", "offset": 8145758208, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 8155195392, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 8202381312, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 8202387456, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.4.attn_q_b.weight", "offset": 8206811136, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 8228044800, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 8228065280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 9093140480, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 12 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 9800929280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 10508718080, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 10508738560, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 10508759040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 11373834240, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 12 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 12081623040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 12789411840, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 12792688640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 12803502080, "shape": [ 5120, 3072 ], "size": 10813440, "type": 12 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 12812349440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 12821196800, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 12821217280, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 12821219328, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.5.attn_kv_b.weight", "offset": 12822878208, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 12832315392, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 12879501312, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 12879507456, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.5.attn_q_b.weight", "offset": 12883931136, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 12905164800, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 12908441600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 12919255040, "shape": [ 5120, 3072 ], "size": 10813440, "type": 12 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 12928102400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 12936949760, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 12936951808, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_kv_b.weight", "offset": 12938610688, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 12948047872, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 12995233792, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 12995239936, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.6.attn_q_b.weight", "offset": 12999663616, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 13020897280, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 13020917760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 13885992960, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 12 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 14593781760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 15301570560, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 15301591040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 15304867840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 15313715200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 15322562560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 15331409920, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 15331411968, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.7.attn_kv_b.weight", "offset": 15333070848, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 15342508032, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 15389693952, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 15389700096, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.7.attn_q_b.weight", "offset": 15394123776, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 15415357440, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 15415377920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 16123166720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 16830955520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 17538744320, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 17538764800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 17542041600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 17550888960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 17559736320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 17568583680, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 17568585728, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.8.attn_kv_b.weight", "offset": 17570244608, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 17579681792, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 17626867712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 17626873856, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.8.attn_q_b.weight", "offset": 17631297536, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 17652531200, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 17652551680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 18360340480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 19068129280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 19775918080, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 19775938560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 19779215360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 19788062720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 19796910080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 19805757440, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 19805759488, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.9.attn_kv_b.weight", "offset": 19807418368, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 19816855552, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 19864041472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 19864047616, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.9.attn_q_b.weight", "offset": 19868471296, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 19889704960, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 19892981760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 19901829120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 19910676480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 19919523840, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 19919525888, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.10.attn_kv_b.weight", "offset": 19921184768, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 19930621952, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 19977807872, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 19977814016, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.10.attn_q_b.weight", "offset": 19982237696, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 20003471360, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 20003491840, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 20711280640, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 21419069440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 22126858240, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 22126878720, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 22126899200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 22834688000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 23542476800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 24250265600, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 24250286080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 24253562880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 24262410240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 24271257600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 24280104960, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 24280107008, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.11.attn_kv_b.weight", "offset": 24281765888, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 24291203072, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 24338388992, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 24338395136, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.11.attn_q_b.weight", "offset": 24342818816, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 24364052480, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 24364072960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 25071861760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 25779650560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 26487439360, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 26487459840, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 26490736640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 26499584000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 26508431360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 26517278720, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 26517280768, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.12.attn_kv_b.weight", "offset": 26518939648, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 26528376832, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 26575562752, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 26575568896, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.12.attn_q_b.weight", "offset": 26579992576, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 26601226240, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 26601246720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 27309035520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 28016824320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 28724613120, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 28724633600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 28727910400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 28736757760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 28745605120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 28754452480, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 28754454528, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.13.attn_kv_b.weight", "offset": 28756113408, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 28765550592, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 28812736512, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 28812742656, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.13.attn_q_b.weight", "offset": 28817166336, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 28838400000, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 28838420480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 29546209280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 30253998080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 30961786880, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 30961807360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 30965084160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 30973931520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 30982778880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 30991626240, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 30991628288, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.14.attn_kv_b.weight", "offset": 30993287168, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 31002724352, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 31049910272, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 31049916416, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.14.attn_q_b.weight", "offset": 31054340096, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 31075573760, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 31075594240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 31783383040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 32491171840, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 33198960640, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 33198981120, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 33202257920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 33211105280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 33219952640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 33228800000, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 33228802048, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.15.attn_kv_b.weight", "offset": 33230460928, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 33239898112, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 33287084032, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 33287090176, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.15.attn_q_b.weight", "offset": 33291513856, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 33312747520, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 33312768000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 34020556800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 34728345600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 35436134400, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 35436154880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 35439431680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 35448279040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 35457126400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 35465973760, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 35465975808, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.16.attn_kv_b.weight", "offset": 35467634688, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 35477071872, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 35524257792, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 35524263936, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.16.attn_q_b.weight", "offset": 35528687616, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 35549921280, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 35549941760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 36257730560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 36965519360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 37673308160, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 37673328640, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 37676605440, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 37685452800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 37694300160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 37703147520, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 37703149568, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.17.attn_kv_b.weight", "offset": 37704808448, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 37714245632, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 37761431552, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 37761437696, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.17.attn_q_b.weight", "offset": 37765861376, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 37787095040, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 37787115520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 38494904320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 39202693120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 39910481920, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 39910502400, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 39910522880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 40618311680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 41326100480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 42033889280, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 42037166080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 42046013440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 42054860800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 42063708160, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 42063728640, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 42063730688, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.18.attn_kv_b.weight", "offset": 42065389568, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 42074826752, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 42122012672, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 42122018816, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.18.attn_q_b.weight", "offset": 42126442496, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 42147676160, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 42150952960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 42159800320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 42168647680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 42177495040, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 42177497088, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.19.attn_kv_b.weight", "offset": 42179155968, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 42188593152, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 42235779072, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 42235785216, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.19.attn_q_b.weight", "offset": 42240208896, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 42261442560, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 42261463040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 42969251840, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 43677040640, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 44384829440, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 44384849920, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 44388126720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 44396974080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 44405821440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 44414668800, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 44414670848, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.20.attn_kv_b.weight", "offset": 44416329728, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 44425766912, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 44472952832, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 44472958976, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.20.attn_q_b.weight", "offset": 44477382656, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 44498616320, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 44498636800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 45206425600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 45914214400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 46622003200, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 46622023680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 46625300480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 46634147840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 46642995200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 46651842560, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "ight\u0001\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\f\ufffd\f\u0000\u0000", "offset": 46651844608, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.21.attn_kv_b.weight", "offset": 46653503488, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 46662940672, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 46710126592, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 46710132736, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.21.attn_q_b.weight", "offset": 46714556416, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 46735790080, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 46735810560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 47443599360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 48151388160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 48859176960, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 48859197440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 48862474240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 48871321600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 48880168960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 48889016320, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 48889018368, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.22.attn_kv_b.weight", "offset": 48890677248, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 48900114432, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 48947300352, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 48947306496, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.22.attn_q_b.weight", "offset": 48951730176, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 48972963840, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 48972984320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 49680773120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 50388561920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 51096350720, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 51096371200, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 51099648000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 51108495360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 51117342720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 51126190080, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 51126192128, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.23.attn_kv_b.weight", "offset": 51127851008, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 51137288192, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 51184474112, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 51184480256, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.23.attn_q_b.weight", "offset": 51188903936, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 51210137600, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 51210158080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 51917946880, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 52625735680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 53333524480, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 53333544960, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 53336821760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 53345669120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 53354516480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 53363363840, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 53363365888, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.24.attn_kv_b.weight", "offset": 53365024768, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 53374461952, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 53421647872, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 53421654016, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.24.attn_q_b.weight", "offset": 53426077696, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 53447311360, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 53447331840, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 54155120640, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 54862909440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 55570698240, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 55570718720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 55573995520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 55582842880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 55591690240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 55600537600, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 55600539648, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.25.attn_kv_b.weight", "offset": 55602198528, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 55611635712, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 55658821632, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 55658827776, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.25.attn_q_b.weight", "offset": 55663251456, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 55684485120, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 55684505600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 56392294400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 57100083200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 57807872000, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 57807892480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 57811169280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 57820016640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 57828864000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 57837711360, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 57837713408, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.26.attn_kv_b.weight", "offset": 57839372288, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 57848809472, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 57895995392, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 57896001536, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.26.attn_q_b.weight", "offset": 57900425216, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 57921658880, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 57921679360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 58629468160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 59337256960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 60045045760, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 60045066240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 60048343040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 60057190400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 60066037760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 60074885120, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 60074887168, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.27.attn_kv_b.weight", "offset": 60076546048, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 60085983232, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 60133169152, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 60133175296, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.27.attn_q_b.weight", "offset": 60137598976, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 60158832640, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 60158853120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 60866641920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 61574430720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 62282219520, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 62282240000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 62285516800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 62294364160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 62303211520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 62312058880, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 62312060928, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.28.attn_kv_b.weight", "offset": 62313719808, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 62323156992, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 62370342912, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 62370349056, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.28.attn_q_b.weight", "offset": 62374772736, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 62396006400, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 62396026880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 63103815680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 63811604480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 64519393280, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 64519413760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 64522690560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 64531537920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "hexp.weight\u0002\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000", "offset": 64540385280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 64549232640, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 64549234688, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.29.attn_kv_b.weight", "offset": 64550893568, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 64560330752, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 64607516672, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 64607522816, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.29.attn_q_b.weight", "offset": 64611946496, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 64633180160, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 64633200640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 65340989440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 66048778240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 66756567040, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 66756587520, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 66759864320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 66768711680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 66777559040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 66786406400, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 66786408448, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.30.attn_kv_b.weight", "offset": 66788067328, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 66797504512, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 66844690432, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 66844696576, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.30.attn_q_b.weight", "offset": 66849120256, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 66870353920, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 66870374400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 67578163200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 68285952000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 68993740800, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 68993761280, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 68993781760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 69701570560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 70409359360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 71117148160, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 71120424960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 71129272320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 71138119680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 71146967040, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 71146987520, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 71146989568, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.31.attn_kv_b.weight", "offset": 71148648448, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.31.attn_output.weight", "offset": 71158085632, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 71205271552, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 71205277696, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.31.attn_q_b.weight", "offset": 71209701376, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 71230935040, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 71234211840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 71243059200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 71251906560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 71260753920, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 71260755968, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.32.attn_kv_b.weight", "offset": 71262414848, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 71271852032, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 71319037952, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 71319044096, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.32.attn_q_b.weight", "offset": 71323467776, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 71344701440, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 71344721920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 72052510720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 72760299520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 73468088320, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 73468108800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 73471385600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 73480232960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 73489080320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 73497927680, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 73497929728, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.33.attn_kv_b.weight", "offset": 73499588608, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 73509025792, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 73556211712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 73556217856, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.33.attn_q_b.weight", "offset": 73560641536, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 73581875200, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 73581895680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 74289684480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 74997473280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 75705262080, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 75705282560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 75708559360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 75717406720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 75726254080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 75735101440, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 75735103488, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.34.attn_kv_b.weight", "offset": 75736762368, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 75746199552, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 75793385472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 75793391616, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.34.attn_q_b.weight", "offset": 75797815296, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 75819048960, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 75819069440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 76526858240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 77234647040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 77942435840, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 77942456320, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 77945733120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 77954580480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 77963427840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 77972275200, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 77972277248, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.35.attn_kv_b.weight", "offset": 77973936128, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 77983373312, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 78030559232, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 78030565376, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.35.attn_q_b.weight", "offset": 78034989056, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 78056222720, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 78056243200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 78764032000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 79471820800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 80179609600, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 80179630080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 80182906880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 80191754240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 80200601600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 80209448960, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 80209451008, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.36.attn_kv_b.weight", "offset": 80211109888, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 80220547072, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 80267732992, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 80267739136, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.36.attn_q_b.weight", "offset": 80272162816, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.36.attn_norm.weight", "offset": 80293396480, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 80293416960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 81001205760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 81708994560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 82416783360, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 82416803840, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 82420080640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 82428928000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 82437775360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 82446622720, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 82446624768, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.37.attn_kv_b.weight", "offset": 82448283648, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 82457720832, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 82504906752, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 82504912896, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.37.attn_q_b.weight", "offset": 82509336576, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 82530570240, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 82530590720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 83238379520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 83946168320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 84653957120, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 84653977600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 84657254400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 84666101760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 84674949120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 84683796480, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 84683798528, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.38.attn_kv_b.weight", "offset": 84685457408, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 84694894592, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 84742080512, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 84742086656, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.38.attn_q_b.weight", "offset": 84746510336, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 84767744000, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 84767764480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 85475553280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 86183342080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 86891130880, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 86891151360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 86894428160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 86903275520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 86912122880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 86920970240, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 86920972288, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.39.attn_kv_b.weight", "offset": 86922631168, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 86932068352, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 86979254272, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 86979260416, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.39.attn_q_b.weight", "offset": 86983684096, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 87004917760, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 87004938240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 87712727040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 88420515840, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 89128304640, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 89128325120, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 89131601920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 89140449280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 89149296640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 89158144000, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 89158146048, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.40.attn_kv_b.weight", "offset": 89159804928, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 89169242112, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 89216428032, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 89216434176, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.40.attn_q_b.weight", "offset": 89220857856, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 89242091520, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 89242112000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 89949900800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 90657689600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.40.ffn_norm.weight", "offset": 91365478400, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 91365498880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 91368775680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 91377623040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 91386470400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 91395317760, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 91395319808, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.41.attn_kv_b.weight", "offset": 91396978688, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 91406415872, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 91453601792, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 91453607936, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.41.attn_q_b.weight", "offset": 91458031616, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 91479265280, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 91479285760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 92187074560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 92894863360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.41.ffn_norm.weight", "offset": 93602652160, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 93602672640, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 93605949440, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 93614796800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 93623644160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 93632491520, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 93632493568, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.42.attn_kv_b.weight", "offset": 93634152448, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.42.attn_output.weight", "offset": 93643589632, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 93690775552, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 93690781696, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.42.attn_q_b.weight", "offset": 93695205376, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.42.attn_norm.weight", "offset": 93716439040, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 93716459520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 94424248320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 95132037120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.42.ffn_norm.weight", "offset": 95839825920, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 95839846400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 95843123200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 95851970560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 95860817920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 95869665280, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 95869667328, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.43.attn_kv_b.weight", "offset": 95871326208, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.43.attn_output.weight", "offset": 95880763392, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 95927949312, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 95927955456, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.43.attn_q_b.weight", "offset": 95932379136, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.43.attn_norm.weight", "offset": 95953612800, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 95953633280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 96661422080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 97369210880, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.43.ffn_norm.weight", "offset": 98076999680, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 98077020160, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 98077040640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 98784829440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 99492618240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 100200407040, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 100203683840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 100212531200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 100221378560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.44.ffn_norm.weight", "offset": 100230225920, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 100230246400, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 100230248448, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.44.attn_kv_b.weight", "offset": 100231907328, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.44.attn_output.weight", "offset": 100241344512, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 100288530432, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 100288536576, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.44.attn_q_b.weight", "offset": 100292960256, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 100314193920, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 100317470720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 100326318080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 100335165440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 100344012800, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 100344014848, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.45.attn_kv_b.weight", "offset": 100345673728, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.45.attn_output.weight", "offset": 100355110912, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 100402296832, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 100402302976, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.45.attn_q_b.weight", "offset": 100406726656, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.45.attn_norm.weight", "offset": 100427960320, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 100427980800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 101135769600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 101843558400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.45.ffn_norm.weight", "offset": 102551347200, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 102551367680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 102554644480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 102563491840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 102572339200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 102581186560, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 102581188608, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.46.attn_kv_b.weight", "offset": 102582847488, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.46.attn_output.weight", "offset": 102592284672, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 102639470592, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 102639476736, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.46.attn_q_b.weight", "offset": 102643900416, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.46.attn_norm.weight", "offset": 102665134080, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 102665154560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 103372943360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 104080732160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.46.ffn_norm.weight", "offset": 104788520960, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 104788541440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 104791818240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 104800665600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 104809512960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 104818360320, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 104818362368, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.47.attn_kv_b.weight", "offset": 104820021248, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.47.attn_output.weight", "offset": 104829458432, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 104876644352, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 104876650496, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.47.attn_q_b.weight", "offset": 104881074176, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.47.attn_norm.weight", "offset": 104902307840, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 104902328320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 105610117120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 106317905920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.47.ffn_norm.weight", "offset": 107025694720, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 107025715200, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 107028992000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 107037839360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 107046686720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 107055534080, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 107055536128, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.48.attn_kv_b.weight", "offset": 107057195008, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.48.attn_output.weight", "offset": 107066632192, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 107113818112, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 107113824256, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.48.attn_q_b.weight", "offset": 107118247936, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.48.attn_norm.weight", "offset": 107139481600, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 107139502080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 107847290880, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 108555079680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.48.ffn_norm.weight", "offset": 109262868480, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 109262888960, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 109266165760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 109275013120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 109283860480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 109292707840, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 109292709888, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.49.attn_kv_b.weight", "offset": 109294368768, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.49.attn_output.weight", "offset": 109303805952, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 109350991872, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 109350998016, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.49.attn_q_b.weight", "offset": 109355421696, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.49.attn_norm.weight", "offset": 109376655360, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 109376675840, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 110084464640, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 110792253440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.49.ffn_norm.weight", "offset": 111500042240, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 111500062720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 111503339520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 111512186880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 111521034240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 111529881600, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 111529883648, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.50.attn_kv_b.weight", "offset": 111531542528, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.50.attn_output.weight", "offset": 111540979712, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 111588165632, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 111588171776, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.50.attn_q_b.weight", "offset": 111592595456, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.50.attn_norm.weight", "offset": 111613829120, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 111613849600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 112321638400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 113029427200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.50.ffn_norm.weight", "offset": 113737216000, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 113737236480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 113740513280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 113749360640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 113758208000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 113767055360, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 113767057408, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.51.attn_kv_b.weight", "offset": 113768716288, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.51.attn_output.weight", "offset": 113778153472, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 113825339392, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 113825345536, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.51.attn_q_b.weight", "offset": 113829769216, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.51.attn_norm.weight", "offset": 113851002880, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 113851023360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 114558812160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 115266600960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.51.ffn_norm.weight", "offset": 115974389760, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 115974410240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 115977687040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 115986534400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 115995381760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 116004229120, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 116004231168, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.52.attn_kv_b.weight", "offset": 116005890048, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.52.attn_output.weight", "offset": 116015327232, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 116062513152, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 116062519296, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.52.attn_q_b.weight", "offset": 116066942976, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.52.attn_norm.weight", "offset": 116088176640, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 116088197120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 116795985920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 117503774720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.52.ffn_norm.weight", "offset": 118211563520, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 118211584000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 118214860800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 118223708160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 118232555520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 118241402880, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 118241404928, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.53.attn_kv_b.weight", "offset": 118243063808, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.53.attn_output.weight", "offset": 118252500992, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 118299686912, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 118299693056, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.53.attn_q_b.weight", "offset": 118304116736, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.53.attn_norm.weight", "offset": 118325350400, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 118325370880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 119033159680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 119740948480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.53.ffn_norm.weight", "offset": 120448737280, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 120448757760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 120452034560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 120460881920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 120469729280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 120478576640, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 120478578688, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.54.attn_kv_b.weight", "offset": 120480237568, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.54.attn_output.weight", "offset": 120489674752, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 120536860672, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 120536866816, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.54.attn_q_b.weight", "offset": 120541290496, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.54.attn_norm.weight", "offset": 120562524160, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 120562544640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 121270333440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 121978122240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.54.ffn_norm.weight", "offset": 122685911040, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 122685931520, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 122689208320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 122698055680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 122706903040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 122715750400, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 122715752448, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.55.attn_kv_b.weight", "offset": 122717411328, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.55.attn_output.weight", "offset": 122726848512, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 122774034432, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 122774040576, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.55.attn_q_b.weight", "offset": 122778464256, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.55.attn_norm.weight", "offset": 122799697920, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 122799718400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 123507507200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 124215296000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.55.ffn_norm.weight", "offset": 124923084800, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 124923105280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 124926382080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 124935229440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 124944076800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 124952924160, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 124952926208, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.56.attn_kv_b.weight", "offset": 124954585088, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.56.attn_output.weight", "offset": 124964022272, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 125011208192, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 125011214336, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.56.attn_q_b.weight", "offset": 125015638016, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.56.attn_norm.weight", "offset": 125036871680, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 125036892160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 125744680960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 126452469760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.56.ffn_norm.weight", "offset": 127160258560, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 127160279040, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 127160299520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 127868088320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 128575877120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 129283665920, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 129286942720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 129295790080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 129304637440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.57.ffn_norm.weight", "offset": 129313484800, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 129313505280, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 129313507328, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.57.attn_kv_b.weight", "offset": 129315166208, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.57.attn_output.weight", "offset": 129324603392, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 129371789312, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 129371795456, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.57.attn_q_b.weight", "offset": 129376219136, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 129397452800, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 129400729600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 129409576960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 129418424320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 129427271680, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 129427273728, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.58.attn_kv_b.weight", "offset": 129428932608, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.58.attn_output.weight", "offset": 129438369792, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 129485555712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 129485561856, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.58.attn_q_b.weight", "offset": 129489985536, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.58.attn_norm.weight", "offset": 129511219200, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 129511239680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 130219028480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 130926817280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.58.ffn_norm.weight", "offset": 131634606080, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 131634626560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 131637903360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 131646750720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 131655598080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 12 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 131664445440, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 131664447488, "shape": [ 5120, 576 ], "size": 2048, "type": 12 }, { "name": "blk.59.attn_kv_b.weight", "offset": 131666106368, "shape": [ 512, 32768 ], "size": 1658880, "type": 12 }, { "name": "blk.59.attn_output.weight", "offset": 131675543552, "shape": [ 16384, 5120 ], "size": 9437184, "type": 12 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 131722729472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 131722735616, "shape": [ 5120, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.59.attn_q_b.weight", "offset": 131727159296, "shape": [ 1536, 24576 ], "size": 4423680, "type": 12 }, { "name": "blk.59.attn_norm.weight", "offset": 131748392960, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 131748413440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 132456202240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 133163991040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 12 }, { "name": "blk.59.ffn_norm.weight", "offset": 133871779840, "shape": [ 5120 ], "size": 707788800, "type": 0 } ], "version": 3 }