deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 13, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 655360000, "shape": [ 5120 ], "size": 225280000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 655380480, "shape": [ 12288, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 698634240, "shape": [ 5120, 12288 ], "size": 43253760, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 725667840, "shape": [ 5120, 12288 ], "size": 27033600, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 752701440, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 752721920, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 752723968, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_kv_b.weight", "offset": 753991168, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 761200128, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 818871808, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 818877952, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.attn_q_b.weight", "offset": 822257152, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 838477312, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 841754112, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 852567552, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 859325952, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 866084352, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 866086400, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000@\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u000b\u0000\u0000\u0000\u0000\ufffd", "offset": 867353600, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 874562560, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 932234240, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 932240384, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.attn_q_b.weight", "offset": 935619584, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "output_norm.weight", "offset": 951839744, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 951860224, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 951880704, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1816955904, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2357627904, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 2898299904, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2898320384, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 2901597184, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 2912410624, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 2919169024, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 2925927424, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 2925929472, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_kv_b.weight", "offset": 2927196672, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 2934405632, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 2992077312, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 2992083456, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.attn_q_b.weight", "offset": 2995462656, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 3011682816, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3011703296, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 3876778496, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 4417450496, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 4958122496, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 4958142976, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 4961419776, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 4972233216, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 4978991616, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 4985750016, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 4985752064, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_kv_b.weight", "offset": 4987019264, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 4994228224, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 5051899904, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 5051906048, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.attn_q_b.weight", "offset": 5055285248, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 5071505408, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 5071525888, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 5936601088, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 6477273088, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 7017945088, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 7017965568, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 7021242368, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 7032055808, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 7038814208, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 7045572608, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 7045574656, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_kv_b.weight", "offset": 7046841856, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 7054050816, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 7111722496, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 7111728640, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.attn_q_b.weight", "offset": 7115107840, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 7131328000, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 7131348480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 7996423680, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 8537095680, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 9077767680, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 9077788160, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 9077808640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 9942883840, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 10483555840, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 11024227840, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 11027504640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 11038318080, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 11045076480, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 11051834880, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 11051855360, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 11051857408, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_kv_b.weight", "offset": 11053124608, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 11060333568, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 11118005248, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 11118011392, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.attn_q_b.weight", "offset": 11121390592, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 11137610752, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 11140887552, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 11151700992, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 11158459392, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 11165217792, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 11165219840, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_kv_b.weight", "offset": 11166487040, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 11173696000, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 11231367680, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 11231373824, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.attn_q_b.weight", "offset": 11234753024, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 11250973184, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 11250993664, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 12116068864, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 12656740864, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 13197412864, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 13197433344, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 13200710144, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 13211523584, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 13218281984, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 13225040384, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 13225042432, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_kv_b.weight", "offset": 13226309632, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 13233518592, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 13291190272, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 13291196416, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.attn_q_b.weight", "offset": 13294575616, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 13310795776, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 13310816256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 14175891456, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 14716563456, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 15257235456, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 15257255936, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 15260532736, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 15271346176, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 15278104576, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 15284862976, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 15284865024, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_kv_b.weight", "offset": 15286132224, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 15293341184, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 15351012864, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 15351019008, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.attn_q_b.weight", "offset": 15354398208, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 15370618368, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 15370638848, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 16235714048, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 16776386048, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 17317058048, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 17317078528, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 17320355328, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 17331168768, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 17337927168, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 17344685568, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 17344687616, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_kv_b.weight", "offset": 17345954816, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 17353163776, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 17410835456, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 17410841600, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.attn_q_b.weight", "offset": 17414220800, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 17430440960, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 17433717760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 17444531200, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 17451289600, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 17458048000, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 17458050048, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_kv_b.weight", "offset": 17459317248, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 17466526208, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 17524197888, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 17524204032, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.attn_q_b.weight", "offset": 17527583232, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 17543803392, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 17543823872, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 18408899072, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 18949571072, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 19490243072, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 19490263552, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 19490284032, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 20355359232, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 20896031232, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 21436703232, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 21436723712, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 21440000512, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 21450813952, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 21457572352, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 21464330752, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 21464332800, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_kv_b.weight", "offset": 21465600000, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 21472808960, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 21530480640, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 21530486784, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.11.attn_q_b.weight", "offset": 21533865984, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 21550086144, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 21550106624, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 22415181824, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 22955853824, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 23496525824, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 23496546304, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 23499823104, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 23510636544, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 23517394944, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 23524153344, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 23524155392, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_kv_b.weight", "offset": 23525422592, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 23532631552, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 23590303232, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 23590309376, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.attn_q_b.weight", "offset": 23593688576, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 23609908736, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 23609929216, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 24475004416, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 25015676416, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 25556348416, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 25556368896, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 25559645696, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 25570459136, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 25577217536, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 25583975936, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 25583977984, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_kv_b.weight", "offset": 25585245184, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 25592454144, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 25650125824, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 25650131968, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.attn_q_b.weight", "offset": 25653511168, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 25669731328, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 25669751808, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 26534827008, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 27075499008, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 27616171008, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 27616191488, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 27619468288, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 27630281728, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 27637040128, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 27643798528, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 27643800576, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_kv_b.weight", "offset": 27645067776, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 27652276736, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 27709948416, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 27709954560, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.attn_q_b.weight", "offset": 27713333760, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 27729553920, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 27729574400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 28594649600, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 29135321600, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 29675993600, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 29676014080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 29679290880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 29690104320, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 29696862720, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 29703621120, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 29703623168, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_kv_b.weight", "offset": 29704890368, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 29712099328, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 29769771008, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 29769777152, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.attn_q_b.weight", "offset": 29773156352, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 29789376512, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 29789396992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 30654472192, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 31195144192, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 31735816192, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 31735836672, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 31739113472, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 31749926912, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 31756685312, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 31763443712, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 31763445760, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_kv_b.weight", "offset": 31764712960, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 31771921920, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 31829593600, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 31829599744, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.attn_q_b.weight", "offset": 31832978944, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 31849199104, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 31849219584, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 32714294784, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 33254966784, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 33795638784, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 33795659264, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 33798936064, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 33809749504, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 33816507904, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 33823266304, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 33823268352, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_kv_b.weight", "offset": 33824535552, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 33831744512, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 33889416192, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 33889422336, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.attn_q_b.weight", "offset": 33892801536, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 33909021696, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 33909042176, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 34774117376, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 35314789376, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 35855461376, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 35855481856, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 35855502336, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 36720577536, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 37261249536, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 37801921536, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 37805198336, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 37816011776, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 37822770176, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 37829528576, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 37829549056, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 37829551104, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_kv_b.weight", "offset": 37830818304, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 37838027264, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 37895698944, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 37895705088, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.attn_q_b.weight", "offset": 37899084288, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 37915304448, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 37918581248, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 37929394688, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 37936153088, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 37942911488, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 37942913536, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_kv_b.weight", "offset": 37944180736, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 37951389696, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 38009061376, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 38009067520, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.attn_q_b.weight", "offset": 38012446720, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 38028666880, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 38028687360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 38893762560, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 39434434560, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 39975106560, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 39975127040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 39978403840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 39989217280, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 39995975680, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 40002734080, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 40002736128, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_kv_b.weight", "offset": 40004003328, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 40011212288, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 40068883968, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 40068890112, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.attn_q_b.weight", "offset": 40072269312, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 40088489472, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 40088509952, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 40953585152, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 41494257152, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 42034929152, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 42034949632, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 42038226432, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 42049039872, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 42055798272, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 42062556672, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 42062558720, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_kv_b.weight", "offset": 42063825920, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 42071034880, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 42128706560, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 42128712704, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.attn_q_b.weight", "offset": 42132091904, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 42148312064, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 42148332544, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 43013407744, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 43554079744, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 44094751744, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 44094772224, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 44098049024, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 44108862464, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 44115620864, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 44122379264, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 44122381312, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_kv_b.weight", "offset": 44123648512, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 44130857472, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 44188529152, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 44188535296, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.attn_q_b.weight", "offset": 44191914496, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 44208134656, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 44208155136, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 45073230336, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 45613902336, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 46154574336, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 46154594816, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 46157871616, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 46168685056, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 46175443456, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 46182201856, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 46182203904, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_kv_b.weight", "offset": 46183471104, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 46190680064, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 46248351744, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 46248357888, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.attn_q_b.weight", "offset": 46251737088, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 46267957248, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 46267977728, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 47133052928, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 47673724928, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 48214396928, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 48214417408, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 48217694208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 48228507648, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 48235266048, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 48242024448, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 48242026496, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_kv_b.weight", "offset": 48243293696, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 48250502656, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 48308174336, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 48308180480, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.attn_q_b.weight", "offset": 48311559680, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 48327779840, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 48327800320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 49192875520, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 49733547520, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 50274219520, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 50274240000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 50277516800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 50288330240, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 50295088640, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 50301847040, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 50301849088, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_kv_b.weight", "offset": 50303116288, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 50310325248, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 50367996928, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 50368003072, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.attn_q_b.weight", "offset": 50371382272, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 50387602432, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 50387622912, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 51252698112, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 51793370112, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 52334042112, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 52334062592, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 52337339392, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 52348152832, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 52354911232, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 52361669632, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 52361671680, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_kv_b.weight", "offset": 52362938880, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 52370147840, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 52427819520, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 52427825664, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.attn_q_b.weight", "offset": 52431204864, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 52447425024, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 52447445504, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 53312520704, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 53853192704, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 54393864704, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 54393885184, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 54397161984, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 54407975424, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 54414733824, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 54421492224, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 54421494272, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.27.attn_kv_b.weight", "offset": 54422761472, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 54429970432, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 54487642112, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 54487648256, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.attn_q_b.weight", "offset": 54491027456, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 54507247616, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 54507268096, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 55372343296, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 55913015296, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 56453687296, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 56453707776, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 56456984576, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 56467798016, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 56474556416, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 56481314816, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 56481316864, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.28.attn_kv_b.weight", "offset": 56482584064, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 56489793024, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 56547464704, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 56547470848, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.28.attn_q_b.weight", "offset": 56550850048, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 56567070208, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 56567090688, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 57432165888, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 57972837888, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 58513509888, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 58513530368, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 58516807168, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 58527620608, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.29.ffn_up_shexp.weight", "offset": 58534379008, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 58541137408, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 58541139456, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.29.attn_kv_b.weight", "offset": 58542406656, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 58549615616, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 58607287296, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 58607293440, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.29.attn_q_b.weight", "offset": 58610672640, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 58626892800, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 58626913280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 59491988480, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 60032660480, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 60573332480, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 60573352960, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 60576629760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 60587443200, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 60594201600, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 60600960000, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 60600962048, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.30.attn_kv_b.weight", "offset": 60602229248, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 60609438208, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 60667109888, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 60667116032, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.30.attn_q_b.weight", "offset": 60670495232, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 60686715392, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 60686735872, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 61551811072, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 62092483072, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 62633155072, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 62633175552, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 62633196032, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 63498271232, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 64038943232, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 64579615232, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 64582892032, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 64593705472, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 64600463872, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 64607222272, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 64607242752, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 64607244800, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.31.attn_kv_b.weight", "offset": 64608512000, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 64615720960, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 64673392640, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 64673398784, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.31.attn_q_b.weight", "offset": 64676777984, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 64692998144, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 64696274944, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 64707088384, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 64713846784, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 64720605184, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 64720607232, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.32.attn_kv_b.weight", "offset": 64721874432, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 64729083392, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 64786755072, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 64786761216, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.32.attn_q_b.weight", "offset": 64790140416, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 64806360576, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 64806381056, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 65671456256, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 66212128256, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 66752800256, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 66752820736, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 66756097536, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 66766910976, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 66773669376, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 66780427776, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 66780429824, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.33.attn_kv_b.weight", "offset": 66781697024, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 66788905984, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 66846577664, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 66846583808, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.33.attn_q_b.weight", "offset": 66849963008, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 66866183168, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 66866203648, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 67731278848, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 68271950848, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 68812622848, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 68812643328, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 68815920128, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 68826733568, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 68833491968, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 68840250368, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 68840252416, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.34.attn_kv_b.weight", "offset": 68841519616, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 68848728576, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 68906400256, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 68906406400, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.34.attn_q_b.weight", "offset": 68909785600, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 68926005760, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 68926026240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 69791101440, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 70331773440, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 70872445440, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 70872465920, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 70875742720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 70886556160, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 70893314560, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 70900072960, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 70900075008, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.35.attn_kv_b.weight", "offset": 70901342208, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 70908551168, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 70966222848, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 70966228992, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.35.attn_q_b.weight", "offset": 70969608192, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 70985828352, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 70985848832, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 71850924032, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 72391596032, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 72932268032, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 72932288512, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 72935565312, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 72946378752, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 72953137152, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 72959895552, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 72959897600, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.36.attn_kv_b.weight", "offset": 72961164800, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 72968373760, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 73026045440, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 73026051584, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.36.attn_q_b.weight", "offset": 73029430784, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 73045650944, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 73045671424, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 73910746624, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 74451418624, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 74992090624, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 74992111104, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 74995387904, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 75006201344, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 75012959744, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 75019718144, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 75019720192, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.37.attn_kv_b.weight", "offset": 75020987392, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 75028196352, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 75085868032, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 75085874176, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.37.attn_q_b.weight", "offset": 75089253376, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 75105473536, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 75105494016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 75970569216, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 76511241216, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 77051913216, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 77051933696, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 77055210496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 77066023936, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 77072782336, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 77079540736, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 77079542784, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.38.attn_kv_b.weight", "offset": 77080809984, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 77088018944, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 77145690624, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 77145696768, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.38.attn_q_b.weight", "offset": 77149075968, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 77165296128, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 77165316608, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 78030391808, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 78571063808, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 79111735808, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 79111756288, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 79115033088, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 79125846528, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 79132604928, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 79139363328, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 79139365376, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.39.attn_kv_b.weight", "offset": 79140632576, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 79147841536, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 79205513216, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 79205519360, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.39.attn_q_b.weight", "offset": 79208898560, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 79225118720, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 79225139200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 80090214400, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 80630886400, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 81171558400, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 81171578880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 81174855680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 81185669120, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 81192427520, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 81199185920, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 81199187968, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.40.attn_kv_b.weight", "offset": 81200455168, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 81207664128, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 81265335808, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 81265341952, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.40.attn_q_b.weight", "offset": 81268721152, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 81284941312, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 81284961792, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 82150036992, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 82690708992, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 83231380992, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 83231401472, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 83234678272, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 83245491712, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 83252250112, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 83259008512, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 83259010560, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.41.attn_kv_b.weight", "offset": 83260277760, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 83267486720, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 83325158400, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 83325164544, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.41.attn_q_b.weight", "offset": 83328543744, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 83344763904, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 83344784384, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 84209859584, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 84750531584, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 85291203584, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 85291224064, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 85294500864, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 85305314304, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 85312072704, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 85318831104, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 85318833152, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.42.attn_kv_b.weight", "offset": 85320100352, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 85327309312, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 85384980992, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 85384987136, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.42.attn_q_b.weight", "offset": 85388366336, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.42.attn_norm.weight", "offset": 85404586496, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 85404606976, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 86269682176, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 86810354176, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 87351026176, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 87351046656, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 87354323456, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 87365136896, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 87371895296, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 87378653696, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 87378655744, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.43.attn_kv_b.weight", "offset": 87379922944, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 87387131904, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 87444803584, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 87444809728, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.43.attn_q_b.weight", "offset": 87448188928, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 87464409088, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 87464429568, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 88329504768, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 88870176768, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 89410848768, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 89410869248, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 89410889728, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 90275964928, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 90816636928, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 91357308928, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 91360585728, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 91371399168, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 91378157568, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 91384915968, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 91384936448, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 91384938496, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.44.attn_kv_b.weight", "offset": 91386205696, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 91393414656, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 91451086336, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 91451092480, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.44.attn_q_b.weight", "offset": 91454471680, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 91470691840, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 91473968640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 91484782080, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 91491540480, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 91498298880, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 91498300928, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.45.attn_kv_b.weight", "offset": 91499568128, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 91506777088, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 91564448768, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 91564454912, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.45.attn_q_b.weight", "offset": 91567834112, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 91584054272, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 91584074752, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 92449149952, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 92989821952, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 93530493952, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 93530514432, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 93533791232, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 93544604672, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 93551363072, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 93558121472, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 93558123520, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.46.attn_kv_b.weight", "offset": 93559390720, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 93566599680, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 93624271360, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 93624277504, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.46.attn_q_b.weight", "offset": 93627656704, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.46.attn_norm.weight", "offset": 93643876864, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 93643897344, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 94508972544, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 95049644544, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 95590316544, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 95590337024, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 95593613824, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 95604427264, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 95611185664, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 95617944064, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 95617946112, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.47.attn_kv_b.weight", "offset": 95619213312, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 95626422272, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 95684093952, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 95684100096, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.47.attn_q_b.weight", "offset": 95687479296, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.47.attn_norm.weight", "offset": 95703699456, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 95703719936, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 96568795136, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 97109467136, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 97650139136, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 97650159616, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 97653436416, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 97664249856, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 97671008256, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 97677766656, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 97677768704, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.48.attn_kv_b.weight", "offset": 97679035904, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 97686244864, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 97743916544, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 97743922688, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.48.attn_q_b.weight", "offset": 97747301888, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.48.attn_norm.weight", "offset": 97763522048, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "\u0000\u0000blk.56.attn_q_b.weight\u0002\u0000\u0000", "offset": 97763542528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 98628617728, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 99169289728, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 99709961728, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 99709982208, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 99713259008, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 99724072448, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 99730830848, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 99737589248, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 99737591296, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.49.attn_kv_b.weight", "offset": 99738858496, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 99746067456, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 99803739136, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 99803745280, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.49.attn_q_b.weight", "offset": 99807124480, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.49.attn_norm.weight", "offset": 99823344640, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 99823365120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 100688440320, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 101229112320, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 101769784320, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 101769804800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 101773081600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 101783895040, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 101790653440, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 101797411840, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 101797413888, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.50.attn_kv_b.weight", "offset": 101798681088, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.50.attn_output.weight", "offset": 101805890048, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 101863561728, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 101863567872, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.50.attn_q_b.weight", "offset": 101866947072, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.50.attn_norm.weight", "offset": 101883167232, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 101883187712, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 102748262912, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 103288934912, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 103829606912, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 103829627392, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 103832904192, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 103843717632, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 103850476032, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 103857234432, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 103857236480, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.51.attn_kv_b.weight", "offset": 103858503680, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 103865712640, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 103923384320, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 103923390464, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.51.attn_q_b.weight", "offset": 103926769664, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.51.attn_norm.weight", "offset": 103942989824, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 103943010304, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 104808085504, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 105348757504, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 105889429504, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 105889449984, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 105892726784, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 105903540224, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 105910298624, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 105917057024, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 105917059072, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.52.attn_kv_b.weight", "offset": 105918326272, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 105925535232, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 105983206912, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 105983213056, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.52.attn_q_b.weight", "offset": 105986592256, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.52.attn_norm.weight", "offset": 106002812416, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 106002832896, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 106867908096, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 107408580096, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 107949252096, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 107949272576, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 107952549376, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 107963362816, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 107970121216, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 107976879616, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 107976881664, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.53.attn_kv_b.weight", "offset": 107978148864, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 107985357824, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 108043029504, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 108043035648, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.53.attn_q_b.weight", "offset": 108046414848, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.53.attn_norm.weight", "offset": 108062635008, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 108062655488, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 108927730688, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 109468402688, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 110009074688, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 110009095168, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 110012371968, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 110023185408, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 110029943808, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 110036702208, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 110036704256, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.54.attn_kv_b.weight", "offset": 110037971456, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 110045180416, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 110102852096, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 110102858240, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.54.attn_q_b.weight", "offset": 110106237440, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.54.attn_norm.weight", "offset": 110122457600, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 110122478080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 110987553280, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 111528225280, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.54.ffn_norm.weight", "offset": 112068897280, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 112068917760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 112072194560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 112083008000, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 112089766400, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 112096524800, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 112096526848, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.55.attn_kv_b.weight", "offset": 112097794048, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 112105003008, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 112162674688, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 112162680832, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.55.attn_q_b.weight", "offset": 112166060032, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.55.attn_norm.weight", "offset": 112182280192, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 112182300672, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 113047375872, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 113588047872, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 114128719872, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 114128740352, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 114132017152, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 114142830592, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 114149588992, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 114156347392, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 114156349440, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.56.attn_kv_b.weight", "offset": 114157616640, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.56.attn_output.weight", "offset": 114164825600, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 114222497280, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 114222503424, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.56.attn_q_b.weight", "offset": 114225882624, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.56.attn_norm.weight", "offset": 114242102784, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 114242123264, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 115107198464, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 115647870464, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.56.ffn_norm.weight", "offset": 116188542464, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 116188562944, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 116188583424, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 117053658624, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 117594330624, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 118135002624, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 118138279424, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 118149092864, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 118155851264, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.57.ffn_norm.weight", "offset": 118162609664, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 118162630144, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 118162632192, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.57.attn_kv_b.weight", "offset": 118163899392, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.57.attn_output.weight", "offset": 118171108352, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 118228780032, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 118228786176, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.57.attn_q_b.weight", "offset": 118232165376, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 118248385536, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 118251662336, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 118262475776, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 118269234176, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 118275992576, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 118275994624, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.58.attn_kv_b.weight", "offset": 118277261824, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.58.attn_output.weight", "offset": 118284470784, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 118342142464, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 118342148608, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.58.attn_q_b.weight", "offset": 118345527808, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.58.attn_norm.weight", "offset": 118361747968, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 118361768448, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 119226843648, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 119767515648, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.58.ffn_norm.weight", "offset": 120308187648, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 120308208128, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 120311484928, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 120322298368, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 120329056768, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 120335815168, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 120335817216, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.59.attn_kv_b.weight", "offset": 120337084416, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.59.attn_output.weight", "offset": 120344293376, "shape": [ 16384, 5120 ], "size": 7208960, "type": 13 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 120401965056, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 120401971200, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.59.attn_q_b.weight", "offset": 120405350400, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.59.attn_norm.weight", "offset": 120421570560, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 120421591040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 121286666240, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 121827338240, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.59.ffn_norm.weight", "offset": 122368010240, "shape": [ 5120 ], "size": 540672000, "type": 0 } ], "version": 3 }