deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 3, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 3 }, { "name": "blk.0.attn_norm.weight", "offset": 757760000, "shape": [ 5120 ], "size": 327680000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 757780480, "shape": [ 12288, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 797102080, "shape": [ 5120, 12288 ], "size": 39321600, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 836423680, "shape": [ 5120, 12288 ], "size": 39321600, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 875745280, "shape": [ 5120 ], "size": 39321600, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 875765760, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 875767808, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.0.attn_kv_b.weight", "offset": 877611008, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 888096768, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 940525568, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 940531712, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.0.attn_q_b.weight", "offset": 945446912, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 969039872, "shape": [ 5120, 160 ], "size": 23592960, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 972316672, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 982147072, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 991977472, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 1001807872, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 1001809920, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.1.attn_kv_b.weight", "offset": 1003653120, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 1014138880, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 1066567680, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1066573824, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.1.attn_q_b.weight", "offset": 1071489024, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "output_norm.weight", "offset": 1095081984, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1095102464, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1095122944, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1881554944, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2667986944, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 3454418944, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3454439424, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 3457716224, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 3467546624, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 3477377024, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 3487207424, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 3487209472, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.2.attn_kv_b.weight", "offset": 3489052672, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 3499538432, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 3551967232, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 3551973376, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.2.attn_q_b.weight", "offset": 3556888576, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 3580481536, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3580502016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 4366934016, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 5153366016, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 5939798016, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 5939818496, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 5943095296, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 5952925696, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 5962756096, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 5972586496, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 5972588544, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.3.attn_kv_b.weight", "offset": 5974431744, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 5984917504, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 6037346304, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 6037352448, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.3.attn_q_b.weight", "offset": 6042267648, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 6065860608, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 6065881088, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 6852313088, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 7638745088, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 8425177088, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 8425197568, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 8428474368, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 8438304768, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 8448135168, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 8457965568, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 8457967616, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.4.attn_kv_b.weight", "offset": 8459810816, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 8470296576, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 8522725376, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 8522731520, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.4.attn_q_b.weight", "offset": 8527646720, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 8551239680, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 8551260160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 9337692160, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 10124124160, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 10910556160, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 10910576640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 10910597120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 11697029120, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 12483461120, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 13269893120, "shape": [ 5120, 160 ], "size": 786432000, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 13273169920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 13283000320, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 13292830720, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 13302661120, "shape": [ 5120 ], "size": 9830400, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 13302681600, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 13302683648, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.5.attn_kv_b.weight", "offset": 13304526848, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 13315012608, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 13367441408, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 13367447552, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.5.attn_q_b.weight", "offset": 13372362752, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 13395955712, "shape": [ 5120, 160 ], "size": 23592960, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 13399232512, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 13409062912, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 13418893312, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 13428723712, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 13428725760, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.6.attn_kv_b.weight", "offset": 13430568960, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 13441054720, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 13493483520, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 13493489664, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.6.attn_q_b.weight", "offset": 13498404864, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 13521997824, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 13522018304, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 14308450304, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 15094882304, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 15881314304, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 15881334784, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 15884611584, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 15894441984, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 15904272384, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 15914102784, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 15914104832, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.7.attn_kv_b.weight", "offset": 15915948032, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 15926433792, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 15978862592, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 15978868736, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.7.attn_q_b.weight", "offset": 15983783936, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 16007376896, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 16007397376, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 16793829376, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 17580261376, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 18366693376, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 18366713856, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 18369990656, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 18379821056, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 18389651456, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 18399481856, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 18399483904, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.8.attn_kv_b.weight", "offset": 18401327104, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 18411812864, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 18464241664, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 18464247808, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.8.attn_q_b.weight", "offset": 18469163008, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 18492755968, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 18492776448, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 19279208448, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 20065640448, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 20852072448, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 20852092928, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 20855369728, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 20865200128, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 20875030528, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 20884860928, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 20884862976, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.9.attn_kv_b.weight", "offset": 20886706176, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 20897191936, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 20949620736, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 20949626880, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.9.attn_q_b.weight", "offset": 20954542080, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 20978135040, "shape": [ 5120, 160 ], "size": 23592960, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 20981411840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 20991242240, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 21001072640, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 21010903040, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 21010905088, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.10.attn_kv_b.weight", "offset": 21012748288, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 21023234048, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 21075662848, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 21075668992, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.10.attn_q_b.weight", "offset": 21080584192, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 21104177152, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 21104197632, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 21890629632, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 22677061632, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 23463493632, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 23463514112, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 23463534592, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 24249966592, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 25036398592, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 25822830592, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 25822851072, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 25826127872, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 25835958272, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 25845788672, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 25855619072, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 25855621120, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.11.attn_kv_b.weight", "offset": 25857464320, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 25867950080, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 25920378880, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 25920385024, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.11.attn_q_b.weight", "offset": 25925300224, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 25948893184, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 25948913664, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 26735345664, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 27521777664, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 28308209664, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 28308230144, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 28311506944, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 28321337344, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 28331167744, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 28340998144, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 28341000192, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.12.attn_kv_b.weight", "offset": 28342843392, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 28353329152, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 28405757952, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 28405764096, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.12.attn_q_b.weight", "offset": 28410679296, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 28434272256, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 28434292736, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 29220724736, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 30007156736, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 30793588736, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 30793609216, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 30796886016, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 30806716416, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 30816546816, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 30826377216, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 30826379264, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.13.attn_kv_b.weight", "offset": 30828222464, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 30838708224, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 30891137024, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 30891143168, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.13.attn_q_b.weight", "offset": 30896058368, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 30919651328, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 30919671808, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 31706103808, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 32492535808, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 33278967808, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 33278988288, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 33282265088, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 33292095488, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 33301925888, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 33311756288, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 33311758336, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.14.attn_kv_b.weight", "offset": 33313601536, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 33324087296, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 33376516096, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 33376522240, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.14.attn_q_b.weight", "offset": 33381437440, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 33405030400, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 33405050880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 34191482880, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 34977914880, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 35764346880, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 35764367360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 35767644160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 35777474560, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 35787304960, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 35797135360, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 35797137408, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.15.attn_kv_b.weight", "offset": 35798980608, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 35809466368, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 35861895168, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 35861901312, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.15.attn_q_b.weight", "offset": 35866816512, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 35890409472, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 35890429952, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 36676861952, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 37463293952, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 38249725952, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 38249746432, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 38253023232, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 38262853632, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 38272684032, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 38282514432, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 38282516480, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.16.attn_kv_b.weight", "offset": 38284359680, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 38294845440, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 38347274240, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 38347280384, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.16.attn_q_b.weight", "offset": 38352195584, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 38375788544, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 38375809024, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 39162241024, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 39948673024, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 40735105024, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 40735125504, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 40738402304, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 40748232704, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 40758063104, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 40767893504, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 40767895552, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.17.attn_kv_b.weight", "offset": 40769738752, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 40780224512, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 40832653312, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 40832659456, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.17.attn_q_b.weight", "offset": 40837574656, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 40861167616, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 40861188096, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 41647620096, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 42434052096, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 43220484096, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 43220504576, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 43220525056, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 44006957056, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 44793389056, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 45579821056, "shape": [ 5120, 160 ], "size": 786432000, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 45583097856, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 45592928256, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 45602758656, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 45612589056, "shape": [ 5120 ], "size": 9830400, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 45612609536, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 45612611584, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.18.attn_kv_b.weight", "offset": 45614454784, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 45624940544, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 45677369344, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 45677375488, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.18.attn_q_b.weight", "offset": 45682290688, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 45705883648, "shape": [ 5120, 160 ], "size": 23592960, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 45709160448, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 45718990848, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 45728821248, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 45738651648, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 45738653696, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.19.attn_kv_b.weight", "offset": 45740496896, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 45750982656, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 45803411456, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 45803417600, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.19.attn_q_b.weight", "offset": 45808332800, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 45831925760, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 45831946240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 46618378240, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 47404810240, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 48191242240, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 48191262720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 48194539520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 48204369920, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 48214200320, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 48224030720, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 48224032768, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.20.attn_kv_b.weight", "offset": 48225875968, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 48236361728, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 48288790528, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 48288796672, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.20.attn_q_b.weight", "offset": 48293711872, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 48317304832, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 48317325312, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 49103757312, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 49890189312, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 50676621312, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 50676641792, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 50679918592, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 50689748992, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 50699579392, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 50709409792, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 50709411840, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.21.attn_kv_b.weight", "offset": 50711255040, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 50721740800, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 50774169600, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 50774175744, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.21.attn_q_b.weight", "offset": 50779090944, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 50802683904, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 50802704384, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 51589136384, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 52375568384, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 53162000384, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 53162020864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 53165297664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 53175128064, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 53184958464, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 53194788864, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 53194790912, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.22.attn_kv_b.weight", "offset": 53196634112, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 53207119872, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 53259548672, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 53259554816, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.22.attn_q_b.weight", "offset": 53264470016, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 53288062976, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 53288083456, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 54074515456, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 54860947456, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 55647379456, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 55647399936, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 55650676736, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 55660507136, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 55670337536, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 55680167936, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 55680169984, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.23.attn_kv_b.weight", "offset": 55682013184, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 55692498944, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 55744927744, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 55744933888, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.23.attn_q_b.weight", "offset": 55749849088, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 55773442048, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 55773462528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 56559894528, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 57346326528, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 58132758528, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 58132779008, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 58136055808, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 58145886208, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 58155716608, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 58165547008, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 58165549056, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.24.attn_kv_b.weight", "offset": 58167392256, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 58177878016, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 58230306816, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 58230312960, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.24.attn_q_b.weight", "offset": 58235228160, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 58258821120, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 58258841600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 59045273600, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 59831705600, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 60618137600, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 60618158080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 60621434880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 60631265280, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 60641095680, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 60650926080, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 60650928128, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.25.attn_kv_b.weight", "offset": 60652771328, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 60663257088, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 60715685888, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 60715692032, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.25.attn_q_b.weight", "offset": 60720607232, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 60744200192, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 60744220672, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 61530652672, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 62317084672, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 63103516672, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 63103537152, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 63106813952, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 63116644352, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 63126474752, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 63136305152, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 63136307200, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.26.attn_kv_b.weight", "offset": 63138150400, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 63148636160, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 63201064960, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 63201071104, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.26.attn_q_b.weight", "offset": 63205986304, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 63229579264, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 63229599744, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 64016031744, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 64802463744, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 65588895744, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 65588916224, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 65592193024, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 65602023424, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 65611853824, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 65621684224, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 65621686272, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.27.attn_kv_b.weight", "offset": 65623529472, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.27.attn_output.weight", "offset": 65634015232, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 65686444032, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 65686450176, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.27.attn_q_b.weight", "offset": 65691365376, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 65714958336, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 65714978816, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 66501410816, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 67287842816, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 68074274816, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 68074295296, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 68077572096, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 68087402496, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 68097232896, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 68107063296, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 68107065344, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.28.attn_kv_b.weight", "offset": 68108908544, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.28.attn_output.weight", "offset": 68119394304, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 68171823104, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 68171829248, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.28.attn_q_b.weight", "offset": 68176744448, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.28.attn_norm.weight", "offset": 68200337408, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 68200357888, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 68986789888, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 69773221888, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.28.ffn_norm.weight", "offset": 70559653888, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 70559674368, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 70562951168, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 70572781568, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.29.ffn_up_shexp.weight", "offset": 70582611968, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 70592442368, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 70592444416, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.29.attn_kv_b.weight", "offset": 70594287616, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.29.attn_output.weight", "offset": 70604773376, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 70657202176, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 70657208320, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.29.attn_q_b.weight", "offset": 70662123520, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.29.attn_norm.weight", "offset": 70685716480, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 70685736960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 71472168960, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 72258600960, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.29.ffn_norm.weight", "offset": 73045032960, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 73045053440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 73048330240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 73058160640, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 73067991040, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 73077821440, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 73077823488, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.30.attn_kv_b.weight", "offset": 73079666688, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.30.attn_output.weight", "offset": 73090152448, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 73142581248, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 73142587392, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.30.attn_q_b.weight", "offset": 73147502592, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.30.attn_norm.weight", "offset": 73171095552, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 73171116032, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 73957548032, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 74743980032, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.30.ffn_norm.weight", "offset": 75530412032, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 75530432512, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 75530452992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 76316884992, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 77103316992, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 77889748992, "shape": [ 5120, 160 ], "size": 786432000, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 77893025792, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 77902856192, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 77912686592, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.31.ffn_norm.weight", "offset": 77922516992, "shape": [ 5120 ], "size": 9830400, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 77922537472, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 77922539520, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.31.attn_kv_b.weight", "offset": 77924382720, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.31.attn_output.weight", "offset": 77934868480, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 77987297280, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 77987303424, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.31.attn_q_b.weight", "offset": 77992218624, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 78015811584, "shape": [ 5120, 160 ], "size": 23592960, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 78019088384, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 78028918784, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 78038749184, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 78048579584, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 78048581632, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.32.attn_kv_b.weight", "offset": 78050424832, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.32.attn_output.weight", "offset": 78060910592, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 78113339392, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 78113345536, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.32.attn_q_b.weight", "offset": 78118260736, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.32.attn_norm.weight", "offset": 78141853696, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 78141874176, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 78928306176, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 79714738176, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.32.ffn_norm.weight", "offset": 80501170176, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 80501190656, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 80504467456, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 80514297856, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 80524128256, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 80533958656, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 80533960704, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.33.attn_kv_b.weight", "offset": 80535803904, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.33.attn_output.weight", "offset": 80546289664, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 80598718464, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 80598724608, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.33.attn_q_b.weight", "offset": 80603639808, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.33.attn_norm.weight", "offset": 80627232768, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 80627253248, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 81413685248, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 82200117248, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.33.ffn_norm.weight", "offset": 82986549248, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 82986569728, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 82989846528, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 82999676928, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 83009507328, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 83019337728, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 83019339776, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.34.attn_kv_b.weight", "offset": 83021182976, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.34.attn_output.weight", "offset": 83031668736, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 83084097536, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 83084103680, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.34.attn_q_b.weight", "offset": 83089018880, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.34.attn_norm.weight", "offset": 83112611840, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 83112632320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 83899064320, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 84685496320, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.34.ffn_norm.weight", "offset": 85471928320, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 85471948800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 85475225600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 85485056000, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 85494886400, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 85504716800, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 85504718848, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.35.attn_kv_b.weight", "offset": 85506562048, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.35.attn_output.weight", "offset": 85517047808, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 85569476608, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 85569482752, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.35.attn_q_b.weight", "offset": 85574397952, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.35.attn_norm.weight", "offset": 85597990912, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 85598011392, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 86384443392, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 87170875392, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.35.ffn_norm.weight", "offset": 87957307392, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 87957327872, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 87960604672, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 87970435072, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 87980265472, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 87990095872, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 87990097920, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.36.attn_kv_b.weight", "offset": 87991941120, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.36.attn_output.weight", "offset": 88002426880, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 88054855680, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 88054861824, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.36.attn_q_b.weight", "offset": 88059777024, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.36.attn_norm.weight", "offset": 88083369984, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 88083390464, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 88869822464, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 89656254464, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.36.ffn_norm.weight", "offset": 90442686464, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 90442706944, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 90445983744, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 90455814144, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 90465644544, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 90475474944, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 90475476992, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.37.attn_kv_b.weight", "offset": 90477320192, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.37.attn_output.weight", "offset": 90487805952, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 90540234752, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 90540240896, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.37.attn_q_b.weight", "offset": 90545156096, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.37.attn_norm.weight", "offset": 90568749056, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 90568769536, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 91355201536, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 92141633536, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.37.ffn_norm.weight", "offset": 92928065536, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 92928086016, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 92931362816, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 92941193216, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 92951023616, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 92960854016, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 92960856064, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.38.attn_kv_b.weight", "offset": 92962699264, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.38.attn_output.weight", "offset": 92973185024, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 93025613824, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 93025619968, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.38.attn_q_b.weight", "offset": 93030535168, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.38.attn_norm.weight", "offset": 93054128128, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 93054148608, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 93840580608, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 94627012608, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.38.ffn_norm.weight", "offset": 95413444608, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 95413465088, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 95416741888, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 95426572288, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 95436402688, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 95446233088, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 95446235136, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.39.attn_kv_b.weight", "offset": 95448078336, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.39.attn_output.weight", "offset": 95458564096, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 95510992896, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 95510999040, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.39.attn_q_b.weight", "offset": 95515914240, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.39.attn_norm.weight", "offset": 95539507200, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 95539527680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 96325959680, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 97112391680, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.39.ffn_norm.weight", "offset": 97898823680, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 97898844160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 97902120960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 97911951360, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 97921781760, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 97931612160, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 97931614208, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.40.attn_kv_b.weight", "offset": 97933457408, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.40.attn_output.weight", "offset": 97943943168, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 97996371968, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 97996378112, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.40.attn_q_b.weight", "offset": 98001293312, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.40.attn_norm.weight", "offset": 98024886272, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 98024906752, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 98811338752, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 99597770752, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.40.ffn_norm.weight", "offset": 100384202752, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 100384223232, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 100387500032, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 100397330432, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 100407160832, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 100416991232, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 100416993280, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.41.attn_kv_b.weight", "offset": 100418836480, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.41.attn_output.weight", "offset": 100429322240, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 100481751040, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 100481757184, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.41.attn_q_b.weight", "offset": 100486672384, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.41.attn_norm.weight", "offset": 100510265344, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 100510285824, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 101296717824, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 102083149824, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.41.ffn_norm.weight", "offset": 102869581824, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 102869602304, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 102872879104, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 102882709504, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 102892539904, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 102902370304, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 102902372352, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.42.attn_kv_b.weight", "offset": 102904215552, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.42.attn_output.weight", "offset": 102914701312, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 102967130112, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 102967136256, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.42.attn_q_b.weight", "offset": 102972051456, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.42.attn_norm.weight", "offset": 102995644416, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 102995664896, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 103782096896, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 104568528896, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.42.ffn_norm.weight", "offset": 105354960896, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 105354981376, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 105358258176, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 105368088576, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 105377918976, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 105387749376, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 105387751424, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.43.attn_kv_b.weight", "offset": 105389594624, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.43.attn_output.weight", "offset": 105400080384, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 105452509184, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 105452515328, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.43.attn_q_b.weight", "offset": 105457430528, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.43.attn_norm.weight", "offset": 105481023488, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 105481043968, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 106267475968, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 107053907968, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.43.ffn_norm.weight", "offset": 107840339968, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 107840360448, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 107840380928, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 108626812928, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 109413244928, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 110199676928, "shape": [ 5120, 160 ], "size": 786432000, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 110202953728, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 110212784128, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 110222614528, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.44.ffn_norm.weight", "offset": 110232444928, "shape": [ 5120 ], "size": 9830400, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 110232465408, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 110232467456, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.44.attn_kv_b.weight", "offset": 110234310656, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.44.attn_output.weight", "offset": 110244796416, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 110297225216, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 110297231360, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.44.attn_q_b.weight", "offset": 110302146560, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 110325739520, "shape": [ 5120, 160 ], "size": 23592960, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 110329016320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 110338846720, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 110348677120, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 110358507520, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 110358509568, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.45.attn_kv_b.weight", "offset": 110360352768, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.45.attn_output.weight", "offset": 110370838528, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 110423267328, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 110423273472, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.45.attn_q_b.weight", "offset": 110428188672, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.45.attn_norm.weight", "offset": 110451781632, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 110451802112, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 111238234112, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 112024666112, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.45.ffn_norm.weight", "offset": 112811098112, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 112811118592, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 112814395392, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 112824225792, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 112834056192, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 112843886592, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 112843888640, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.46.attn_kv_b.weight", "offset": 112845731840, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.46.attn_output.weight", "offset": 112856217600, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 112908646400, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 112908652544, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.46.attn_q_b.weight", "offset": 112913567744, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.46.attn_norm.weight", "offset": 112937160704, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 112937181184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 113723613184, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 114510045184, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.46.ffn_norm.weight", "offset": 115296477184, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 115296497664, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 115299774464, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 115309604864, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 115319435264, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 115329265664, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 115329267712, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.47.attn_kv_b.weight", "offset": 115331110912, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.47.attn_output.weight", "offset": 115341596672, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 115394025472, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 115394031616, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.47.attn_q_b.weight", "offset": 115398946816, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.47.attn_norm.weight", "offset": 115422539776, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 115422560256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 116208992256, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 116995424256, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.47.ffn_norm.weight", "offset": 117781856256, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 117781876736, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 117785153536, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 117794983936, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 117804814336, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 117814644736, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 117814646784, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.48.attn_kv_b.weight", "offset": 117816489984, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.48.attn_output.weight", "offset": 117826975744, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 117879404544, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 117879410688, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.48.attn_q_b.weight", "offset": 117884325888, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.48.attn_norm.weight", "offset": 117907918848, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 117907939328, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 118694371328, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 119480803328, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.48.ffn_norm.weight", "offset": 120267235328, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 120267255808, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 120270532608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 120280363008, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 120290193408, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 120300023808, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 120300025856, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.49.attn_kv_b.weight", "offset": 120301869056, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.49.attn_output.weight", "offset": 120312354816, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 120364783616, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 120364789760, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.49.attn_q_b.weight", "offset": 120369704960, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.49.attn_norm.weight", "offset": 120393297920, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 120393318400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 121179750400, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 121966182400, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.49.ffn_norm.weight", "offset": 122752614400, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 122752634880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 122755911680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 122765742080, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 122775572480, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 122785402880, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 122785404928, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.50.attn_kv_b.weight", "offset": 122787248128, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.50.attn_output.weight", "offset": 122797733888, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 122850162688, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 122850168832, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.50.attn_q_b.weight", "offset": 122855084032, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.50.attn_norm.weight", "offset": 122878676992, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 122878697472, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 123665129472, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 124451561472, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.50.ffn_norm.weight", "offset": 125237993472, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 125238013952, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 125241290752, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 125251121152, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 125260951552, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 125270781952, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 125270784000, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.51.attn_kv_b.weight", "offset": 125272627200, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.51.attn_output.weight", "offset": 125283112960, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 125335541760, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 125335547904, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.51.attn_q_b.weight", "offset": 125340463104, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.51.attn_norm.weight", "offset": 125364056064, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 125364076544, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 126150508544, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 126936940544, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.51.ffn_norm.weight", "offset": 127723372544, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 127723393024, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 127726669824, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 127736500224, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 127746330624, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 127756161024, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 127756163072, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.52.attn_kv_b.weight", "offset": 127758006272, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.52.attn_output.weight", "offset": 127768492032, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 127820920832, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 127820926976, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.52.attn_q_b.weight", "offset": 127825842176, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.52.attn_norm.weight", "offset": 127849435136, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 127849455616, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 128635887616, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 129422319616, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.52.ffn_norm.weight", "offset": 130208751616, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 130208772096, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 130212048896, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 130221879296, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 130231709696, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 130241540096, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 130241542144, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.53.attn_kv_b.weight", "offset": 130243385344, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.53.attn_output.weight", "offset": 130253871104, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 130306299904, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 130306306048, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.53.attn_q_b.weight", "offset": 130311221248, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.53.attn_norm.weight", "offset": 130334814208, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 130334834688, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 131121266688, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 131907698688, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.53.ffn_norm.weight", "offset": 132694130688, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 132694151168, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 132697427968, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 132707258368, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 132717088768, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 132726919168, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 132726921216, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.54.attn_kv_b.weight", "offset": 132728764416, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.54.attn_output.weight", "offset": 132739250176, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 132791678976, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 132791685120, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.54.attn_q_b.weight", "offset": 132796600320, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.54.attn_norm.weight", "offset": 132820193280, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 132820213760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 133606645760, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 134393077760, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.54.ffn_norm.weight", "offset": 135179509760, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 135179530240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 135182807040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 135192637440, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 135202467840, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 135212298240, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 135212300288, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.55.attn_kv_b.weight", "offset": 135214143488, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.55.attn_output.weight", "offset": 135224629248, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 135277058048, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 135277064192, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.55.attn_q_b.weight", "offset": 135281979392, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.55.attn_norm.weight", "offset": 135305572352, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 135305592832, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 136092024832, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 136878456832, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.55.ffn_norm.weight", "offset": 137664888832, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 137664909312, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 137668186112, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 137678016512, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 137687846912, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 137697677312, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 137697679360, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.56.attn_kv_b.weight", "offset": 137699522560, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.56.attn_output.weight", "offset": 137710008320, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 137762437120, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 137762443264, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.56.attn_q_b.weight", "offset": 137767358464, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.56.attn_norm.weight", "offset": 137790951424, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 137790971904, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 138577403904, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 139363835904, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.56.ffn_norm.weight", "offset": 140150267904, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 140150288384, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 140150308864, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 140936740864, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 141723172864, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 142509604864, "shape": [ 5120, 160 ], "size": 786432000, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 142512881664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 142522712064, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 142532542464, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.57.ffn_norm.weight", "offset": 142542372864, "shape": [ 5120 ], "size": 9830400, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 142542393344, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 142542395392, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.57.attn_kv_b.weight", "offset": 142544238592, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.57.attn_output.weight", "offset": 142554724352, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 142607153152, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 142607159296, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.57.attn_q_b.weight", "offset": 142612074496, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 142635667456, "shape": [ 5120, 160 ], "size": 23592960, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 142638944256, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 142648774656, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 142658605056, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 142668435456, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 142668437504, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.58.attn_kv_b.weight", "offset": 142670280704, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.58.attn_output.weight", "offset": 142680766464, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 142733195264, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 142733201408, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.58.attn_q_b.weight", "offset": 142738116608, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.58.attn_norm.weight", "offset": 142761709568, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 142761730048, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 143548162048, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 144334594048, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.58.ffn_norm.weight", "offset": 145121026048, "shape": [ 5120 ], "size": 786432000, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 145121046528, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 145124323328, "shape": [ 3072, 5120 ], "size": 3276800, "type": 3 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 145134153728, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 145143984128, "shape": [ 5120, 3072 ], "size": 9830400, "type": 3 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 145153814528, "shape": [ 512 ], "size": 9830400, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 145153816576, "shape": [ 5120, 576 ], "size": 2048, "type": 3 }, { "name": "blk.59.attn_kv_b.weight", "offset": 145155659776, "shape": [ 512, 32768 ], "size": 1843200, "type": 3 }, { "name": "blk.59.attn_output.weight", "offset": 145166145536, "shape": [ 16384, 5120 ], "size": 10485760, "type": 3 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 145218574336, "shape": [ 1536 ], "size": 52428800, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 145218580480, "shape": [ 5120, 1536 ], "size": 6144, "type": 3 }, { "name": "blk.59.attn_q_b.weight", "offset": 145223495680, "shape": [ 1536, 24576 ], "size": 4915200, "type": 3 }, { "name": "blk.59.attn_norm.weight", "offset": 145247088640, "shape": [ 5120 ], "size": 23592960, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 145247109120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 3 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 146033541120, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 146819973120, "shape": [ 5120, 1536, 160 ], "size": 786432000, "type": 3 }, { "name": "blk.59.ffn_norm.weight", "offset": 147606405120, "shape": [ 5120 ], "size": 786432000, "type": 0 } ], "version": 3 }