deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 12, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 655360000, "shape": [ 5120 ], "size": 225280000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 655380480, "shape": [ 12288, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 698634240, "shape": [ 5120, 12288 ], "size": 43253760, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 725667840, "shape": [ 5120, 12288 ], "size": 27033600, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 752701440, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 752721920, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 752723968, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_kv_b.weight", "offset": 753991168, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 761200128, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 808386048, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 808392192, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.attn_q_b.weight", "offset": 811771392, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 827991552, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 831268352, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 842081792, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 848840192, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 855598592, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 855600640, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_kv_b.weight", "offset": 856867840, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 864076800, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 911262720, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 911268864, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.attn_q_b.weight", "offset": 914648064, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "output_norm.weight", "offset": 930868224, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 930888704, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 930909184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1795984384, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2336656384, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 2877328384, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2877348864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 2880625664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 2891439104, "shape": [ 5120, 3072 ], "size": 10813440, "type": 11 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 2898197504, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 2904955904, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 2904957952, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_kv_b.weight", "offset": 2906225152, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 2913434112, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 2960620032, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 2960626176, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.attn_q_b.weight", "offset": 2964005376, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 2980225536, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 2980246016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 3845321216, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 4385993216, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 4926665216, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 4926685696, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 4929962496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 4938809856, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 4945568256, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 4952326656, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 4952328704, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_kv_b.weight", "offset": 4953595904, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 4960804864, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 5007990784, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 5007996928, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.attn_q_b.weight", "offset": 5011376128, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 5027596288, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 5027616768, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 5735405568, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 6276077568, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 6816749568, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 6816770048, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 6820046848, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 6828894208, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 6835652608, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 6842411008, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 6842413056, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_kv_b.weight", "offset": 6843680256, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 6850889216, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 6898075136, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 6898081280, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.attn_q_b.weight", "offset": 6901460480, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 6917680640, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 6917701120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 7625489920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 8166161920, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 8706833920, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 8706854400, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 8706874880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 9414663680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 9955335680, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 10496007680, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 10499284480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 10508131840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 10514890240, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 10521648640, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 10521669120, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 10521671168, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_kv_b.weight", "offset": 10522938368, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 10530147328, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 10577333248, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 10577339392, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.attn_q_b.weight", "offset": 10580718592, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 10596938752, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 10600215552, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 10609062912, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 10615821312, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 10622579712, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 10622581760, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_kv_b.weight", "offset": 10623848960, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 10631057920, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 10678243840, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 10678249984, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.attn_q_b.weight", "offset": 10681629184, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 10697849344, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 10697869824, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 11405658624, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 11946330624, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 12487002624, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 12487023104, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 12490299904, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 12499147264, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 12505905664, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 12512664064, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 12512666112, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_kv_b.weight", "offset": 12513933312, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 12521142272, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 12568328192, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 12568334336, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.attn_q_b.weight", "offset": 12571713536, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 12587933696, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 12587954176, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 13295742976, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 13836414976, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 14377086976, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 14377107456, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 14380384256, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 14389231616, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 14395990016, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 14402748416, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 14402750464, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_kv_b.weight", "offset": 14404017664, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 14411226624, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 14458412544, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 14458418688, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.attn_q_b.weight", "offset": 14461797888, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 14478018048, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 14478038528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 15185827328, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 15726499328, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 16267171328, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 16267191808, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 16270468608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 16279315968, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 16286074368, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 16292832768, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 16292834816, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_kv_b.weight", "offset": 16294102016, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 16301310976, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 16348496896, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 16348503040, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.attn_q_b.weight", "offset": 16351882240, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 16368102400, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 16371379200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 16380226560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 16386984960, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 16393743360, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 16393745408, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_kv_b.weight", "offset": 16395012608, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 16402221568, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 16449407488, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 16449413632, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.attn_q_b.weight", "offset": 16452792832, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 16469012992, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 16469033472, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 17176822272, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 17717494272, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 18258166272, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 18258186752, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 18258207232, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 18965996032, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 19506668032, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 20047340032, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 20047360512, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 20050637312, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 20059484672, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 20066243072, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 20073001472, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 20073003520, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_kv_b.weight", "offset": 20074270720, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 20081479680, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 20128665600, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 20128671744, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.11.attn_q_b.weight", "offset": 20132050944, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 20148271104, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 20148291584, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 20856080384, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 21396752384, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 21937424384, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 21937444864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 21940721664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 21949569024, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 21956327424, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 21963085824, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 21963087872, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_kv_b.weight", "offset": 21964355072, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 21971564032, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 22018749952, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 22018756096, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.attn_q_b.weight", "offset": 22022135296, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 22038355456, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 22038375936, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 22746164736, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 23286836736, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 23827508736, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 23827529216, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 23830806016, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 23839653376, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 23846411776, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 23853170176, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 23853172224, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_kv_b.weight", "offset": 23854439424, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 23861648384, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 23908834304, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 23908840448, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.attn_q_b.weight", "offset": 23912219648, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 23928439808, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 23928460288, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 24636249088, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 25176921088, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 25717593088, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 25717613568, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 25720890368, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 25729737728, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 25736496128, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 25743254528, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 25743256576, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_kv_b.weight", "offset": 25744523776, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 25751732736, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 25798918656, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 25798924800, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.attn_q_b.weight", "offset": 25802304000, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 25818524160, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 25818544640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 26526333440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 27067005440, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 27607677440, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 27607697920, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 27610974720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 27619822080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 27626580480, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 27633338880, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 27633340928, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_kv_b.weight", "offset": 27634608128, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 27641817088, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 27689003008, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 27689009152, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.attn_q_b.weight", "offset": 27692388352, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 27708608512, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 27708628992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 28416417792, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 28957089792, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 29497761792, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 29497782272, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 29501059072, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 29509906432, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 29516664832, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 29523423232, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 29523425280, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_kv_b.weight", "offset": 29524692480, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 29531901440, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 29579087360, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 29579093504, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.attn_q_b.weight", "offset": 29582472704, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 29598692864, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 29598713344, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 30306502144, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 30847174144, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 31387846144, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 31387866624, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 31391143424, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 31399990784, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 31406749184, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 31413507584, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 31413509632, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_kv_b.weight", "offset": 31414776832, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 31421985792, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 31469171712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 31469177856, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.attn_q_b.weight", "offset": 31472557056, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 31488777216, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 31488797696, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 32196586496, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 32737258496, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 33277930496, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 33277950976, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 33277971456, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 33985760256, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 34526432256, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 35067104256, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 35070381056, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 35079228416, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 35085986816, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 35092745216, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 35092765696, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 35092767744, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_kv_b.weight", "offset": 35094034944, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 35101243904, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 35148429824, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 35148435968, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.attn_q_b.weight", "offset": 35151815168, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 35168035328, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 35171312128, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 35180159488, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 35186917888, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 35193676288, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 35193678336, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_kv_b.weight", "offset": 35194945536, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 35202154496, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 35249340416, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 35249346560, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.attn_q_b.weight", "offset": 35252725760, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 35268945920, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 35268966400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 35976755200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 36517427200, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 37058099200, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 37058119680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 37061396480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 37070243840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 37077002240, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 37083760640, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 37083762688, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_kv_b.weight", "offset": 37085029888, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 37092238848, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 37139424768, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 37139430912, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.attn_q_b.weight", "offset": 37142810112, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 37159030272, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 37159050752, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 37866839552, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 38407511552, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 38948183552, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 38948204032, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 38951480832, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 38960328192, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 38967086592, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 38973844992, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "\u000b\u0000\u0000\u0000\u0000\ufffd\u0005\u0013\t\u0000\u0000\u0000\u0017\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.21.", "offset": 38973847040, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_kv_b.weight", "offset": 38975114240, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 38982323200, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 39029509120, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 39029515264, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.attn_q_b.weight", "offset": 39032894464, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 39049114624, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 39049135104, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 39756923904, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 40297595904, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 40838267904, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 40838288384, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 40841565184, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 40850412544, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 40857170944, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 40863929344, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 40863931392, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_kv_b.weight", "offset": 40865198592, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 40872407552, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 40919593472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 40919599616, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.attn_q_b.weight", "offset": 40922978816, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 40939198976, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 40939219456, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 41647008256, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 42187680256, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 42728352256, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 42728372736, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 42731649536, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 42740496896, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 42747255296, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 42754013696, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 42754015744, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_kv_b.weight", "offset": 42755282944, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 42762491904, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 42809677824, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 42809683968, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.attn_q_b.weight", "offset": 42813063168, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 42829283328, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 42829303808, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 43537092608, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 44077764608, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 44618436608, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 44618457088, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 44621733888, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 44630581248, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 44637339648, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 44644098048, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 44644100096, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_kv_b.weight", "offset": 44645367296, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 44652576256, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 44699762176, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 44699768320, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.attn_q_b.weight", "offset": 44703147520, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 44719367680, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 44719388160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 45427176960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 45967848960, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 46508520960, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 46508541440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 46511818240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 46520665600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 46527424000, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 46534182400, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 46534184448, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_kv_b.weight", "offset": 46535451648, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 46542660608, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 46589846528, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 46589852672, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.attn_q_b.weight", "offset": 46593231872, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 46609452032, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 46609472512, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 47317261312, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 47857933312, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 48398605312, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 48398625792, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 48401902592, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 48410749952, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 48417508352, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 48424266752, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 48424268800, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_kv_b.weight", "offset": 48425536000, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 48432744960, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 48479930880, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 48479937024, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.attn_q_b.weight", "offset": 48483316224, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 48499536384, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 48499556864, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 49207345664, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 49748017664, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 50288689664, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 50288710144, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 50291986944, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 50300834304, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 50307592704, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 50314351104, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 50314353152, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.27.attn_kv_b.weight", "offset": 50315620352, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 50322829312, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 50370015232, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 50370021376, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.attn_q_b.weight", "offset": 50373400576, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 50389620736, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 50389641216, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 51097430016, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 51638102016, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 52178774016, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 52178794496, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 52182071296, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 52190918656, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 52197677056, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 52204435456, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 52204437504, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.28.attn_kv_b.weight", "offset": 52205704704, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 52212913664, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 52260099584, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 52260105728, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.28.attn_q_b.weight", "offset": 52263484928, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 52279705088, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 52279725568, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 52987514368, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 53528186368, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 54068858368, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 54068878848, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 54072155648, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 54081003008, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "hexp.weight\u0002\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000", "offset": 54087761408, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 54094519808, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 54094521856, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.29.attn_kv_b.weight", "offset": 54095789056, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 54102998016, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 54150183936, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 54150190080, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.29.attn_q_b.weight", "offset": 54153569280, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 54169789440, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 54169809920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 54877598720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 55418270720, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 55958942720, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 55958963200, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 55962240000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 55971087360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 55977845760, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 55984604160, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 55984606208, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.30.attn_kv_b.weight", "offset": 55985873408, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 55993082368, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 56040268288, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 56040274432, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.30.attn_q_b.weight", "offset": 56043653632, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 56059873792, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 56059894272, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 56767683072, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 57308355072, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 57849027072, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 57849047552, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 57849068032, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 58556856832, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 59097528832, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 59638200832, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 59641477632, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 59650324992, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 59657083392, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 59663841792, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 59663862272, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 59663864320, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.31.attn_kv_b.weight", "offset": 59665131520, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 59672340480, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 59719526400, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 59719532544, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.31.attn_q_b.weight", "offset": 59722911744, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 59739131904, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 59742408704, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 59751256064, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 59758014464, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 59764772864, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 59764774912, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.32.attn_kv_b.weight", "offset": 59766042112, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 59773251072, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 59820436992, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 59820443136, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.32.attn_q_b.weight", "offset": 59823822336, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 59840042496, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 59840062976, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 60547851776, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 61088523776, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 61629195776, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 61629216256, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 61632493056, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 61641340416, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 61648098816, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 61654857216, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 61654859264, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.33.attn_kv_b.weight", "offset": 61656126464, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 61663335424, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 61710521344, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 61710527488, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.33.attn_q_b.weight", "offset": 61713906688, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 61730126848, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 61730147328, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 62437936128, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 62978608128, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 63519280128, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 63519300608, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 63522577408, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 63531424768, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 63538183168, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 63544941568, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 63544943616, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.34.attn_kv_b.weight", "offset": 63546210816, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 63553419776, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 63600605696, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 63600611840, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.34.attn_q_b.weight", "offset": 63603991040, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 63620211200, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 63620231680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 64328020480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 64868692480, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 65409364480, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 65409384960, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 65412661760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 65421509120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 65428267520, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 65435025920, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 65435027968, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.35.attn_kv_b.weight", "offset": 65436295168, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 65443504128, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 65490690048, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 65490696192, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.35.attn_q_b.weight", "offset": 65494075392, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 65510295552, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 65510316032, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 66218104832, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 66758776832, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 67299448832, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 67299469312, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 67302746112, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 67311593472, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 67318351872, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 67325110272, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 67325112320, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.36.attn_kv_b.weight", "offset": 67326379520, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 67333588480, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 67380774400, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 67380780544, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.36.attn_q_b.weight", "offset": 67384159744, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 67400379904, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 67400400384, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 68108189184, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 68648861184, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 69189533184, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 69189553664, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 69192830464, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 69201677824, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 69208436224, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 69215194624, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 69215196672, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.37.attn_kv_b.weight", "offset": 69216463872, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 69223672832, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 69270858752, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 69270864896, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.37.attn_q_b.weight", "offset": 69274244096, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 69290464256, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 69290484736, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 69998273536, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 70538945536, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 71079617536, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 71079638016, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 71082914816, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 71091762176, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 71098520576, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 71105278976, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 71105281024, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.38.attn_kv_b.weight", "offset": 71106548224, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 71113757184, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 71160943104, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 71160949248, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.38.attn_q_b.weight", "offset": 71164328448, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 71180548608, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 71180569088, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 71888357888, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 72429029888, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 72969701888, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 72969722368, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 72972999168, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 72981846528, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 72988604928, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 72995363328, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 72995365376, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.39.attn_kv_b.weight", "offset": 72996632576, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 73003841536, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 73051027456, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 73051033600, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.39.attn_q_b.weight", "offset": 73054412800, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 73070632960, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 73070653440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 73778442240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 74319114240, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 74859786240, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 74859806720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 74863083520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 74871930880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 74878689280, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 74885447680, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 74885449728, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.40.attn_kv_b.weight", "offset": 74886716928, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 74893925888, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 74941111808, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 74941117952, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.40.attn_q_b.weight", "offset": 74944497152, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 74960717312, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 74960737792, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 75668526592, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 76209198592, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 76749870592, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 76749891072, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 76753167872, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 76762015232, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 76768773632, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 76775532032, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 76775534080, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.41.attn_kv_b.weight", "offset": 76776801280, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 76784010240, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 76831196160, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 76831202304, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.41.attn_q_b.weight", "offset": 76834581504, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 76850801664, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 76850822144, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 77558610944, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 78099282944, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 78639954944, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 78639975424, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 78643252224, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 78652099584, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 78658857984, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 78665616384, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 78665618432, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.42.attn_kv_b.weight", "offset": 78666885632, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 78674094592, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 78721280512, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 78721286656, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.42.attn_q_b.weight", "offset": 78724665856, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.42.attn_norm.weight", "offset": 78740886016, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 78740906496, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 79448695296, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 79989367296, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 80530039296, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 80530059776, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 80533336576, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 80542183936, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 80548942336, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 80555700736, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 80555702784, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.43.attn_kv_b.weight", "offset": 80556969984, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 80564178944, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 80611364864, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 80611371008, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.43.attn_q_b.weight", "offset": 80614750208, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 80630970368, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 80630990848, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 81338779648, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 81879451648, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 82420123648, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 82420144128, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 82420164608, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 83127953408, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 83668625408, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 84209297408, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 84212574208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 84221421568, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 84228179968, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 84234938368, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 84234958848, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 84234960896, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.44.attn_kv_b.weight", "offset": 84236228096, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 84243437056, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 84290622976, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 84290629120, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.44.attn_q_b.weight", "offset": 84294008320, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 84310228480, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 84313505280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 84322352640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 84329111040, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 84335869440, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 84335871488, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.45.attn_kv_b.weight", "offset": 84337138688, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 84344347648, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 84391533568, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 84391539712, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.45.attn_q_b.weight", "offset": 84394918912, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 84411139072, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 84411159552, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 85118948352, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 85659620352, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 86200292352, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 86200312832, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 86203589632, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 86212436992, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 86219195392, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 86225953792, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 86225955840, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.46.attn_kv_b.weight", "offset": 86227223040, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 86234432000, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 86281617920, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 86281624064, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.46.attn_q_b.weight", "offset": 86285003264, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.46.attn_norm.weight", "offset": 86301223424, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 86301243904, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 87009032704, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 87549704704, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 88090376704, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 88090397184, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 88093673984, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 88102521344, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 88109279744, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 88116038144, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 88116040192, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.47.attn_kv_b.weight", "offset": 88117307392, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 88124516352, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 88171702272, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 88171708416, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.47.attn_q_b.weight", "offset": 88175087616, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.47.attn_norm.weight", "offset": 88191307776, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 88191328256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 88899117056, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 89439789056, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 89980461056, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 89980481536, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 89983758336, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 89992605696, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 89999364096, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 90006122496, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 90006124544, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.48.attn_kv_b.weight", "offset": 90007391744, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 90014600704, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 90061786624, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 90061792768, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.48.attn_q_b.weight", "offset": 90065171968, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.48.attn_norm.weight", "offset": 90081392128, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 90081412608, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 90789201408, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 91329873408, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 91870545408, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 91870565888, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 91873842688, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 91882690048, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 91889448448, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 91896206848, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 91896208896, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.49.attn_kv_b.weight", "offset": 91897476096, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 91904685056, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 91951870976, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 91951877120, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.49.attn_q_b.weight", "offset": 91955256320, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.49.attn_norm.weight", "offset": 91971476480, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 91971496960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 92679285760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 93219957760, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 93760629760, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 93760650240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 93763927040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 93772774400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 93779532800, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 93786291200, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 93786293248, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.50.attn_kv_b.weight", "offset": 93787560448, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.50.attn_output.weight", "offset": 93794769408, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 93841955328, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 93841961472, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.50.attn_q_b.weight", "offset": 93845340672, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.50.attn_norm.weight", "offset": 93861560832, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 93861581312, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 94569370112, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 95110042112, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 95650714112, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 95650734592, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 95654011392, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 95662858752, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 95669617152, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 95676375552, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 95676377600, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.51.attn_kv_b.weight", "offset": 95677644800, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 95684853760, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 95732039680, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 95732045824, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.51.attn_q_b.weight", "offset": 95735425024, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.51.attn_norm.weight", "offset": 95751645184, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 95751665664, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 96459454464, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 97000126464, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 97540798464, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 97540818944, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 97544095744, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 97552943104, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 97559701504, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 97566459904, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 97566461952, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.52.attn_kv_b.weight", "offset": 97567729152, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 97574938112, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 97622124032, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 97622130176, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.52.attn_q_b.weight", "offset": 97625509376, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.52.attn_norm.weight", "offset": 97641729536, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 97641750016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 98349538816, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 98890210816, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 99430882816, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 99430903296, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 99434180096, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 99443027456, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 99449785856, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 99456544256, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 99456546304, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.53.attn_kv_b.weight", "offset": 99457813504, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 99465022464, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 99512208384, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 99512214528, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.53.attn_q_b.weight", "offset": 99515593728, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.53.attn_norm.weight", "offset": 99531813888, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 99531834368, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 100239623168, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 100780295168, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 101320967168, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 101320987648, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 101324264448, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 101333111808, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 101339870208, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 101346628608, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 101346630656, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.54.attn_kv_b.weight", "offset": 101347897856, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 101355106816, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 101402292736, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 101402298880, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.54.attn_q_b.weight", "offset": 101405678080, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.54.attn_norm.weight", "offset": 101421898240, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 101421918720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 102129707520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 102670379520, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.54.ffn_norm.weight", "offset": 103211051520, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 103211072000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 103214348800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 103223196160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 103229954560, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 103236712960, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 103236715008, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.55.attn_kv_b.weight", "offset": 103237982208, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 103245191168, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 103292377088, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 103292383232, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.55.attn_q_b.weight", "offset": 103295762432, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.55.attn_norm.weight", "offset": 103311982592, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 103312003072, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 104019791872, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 104560463872, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 105101135872, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 105101156352, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 105104433152, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 105113280512, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 105120038912, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 105126797312, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 105126799360, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.56.attn_kv_b.weight", "offset": 105128066560, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.56.attn_output.weight", "offset": 105135275520, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 105182461440, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 105182467584, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.56.attn_q_b.weight", "offset": 105185846784, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.56.attn_norm.weight", "offset": 105202066944, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 105202087424, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 105909876224, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 106450548224, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.56.ffn_norm.weight", "offset": 106991220224, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 106991240704, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 106991261184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 107699049984, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 108239721984, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 108780393984, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 108783670784, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 108792518144, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 108799276544, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.57.ffn_norm.weight", "offset": 108806034944, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 108806055424, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 108806057472, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.57.attn_kv_b.weight", "offset": 108807324672, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.57.attn_output.weight", "offset": 108814533632, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 108861719552, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 108861725696, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.57.attn_q_b.weight", "offset": 108865104896, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 108881325056, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 108884601856, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 108893449216, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 108900207616, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 108906966016, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 108906968064, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.58.attn_kv_b.weight", "offset": 108908235264, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.58.attn_output.weight", "offset": 108915444224, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 108962630144, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 108962636288, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.58.attn_q_b.weight", "offset": 108966015488, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.58.attn_norm.weight", "offset": 108982235648, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 108982256128, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 109690044928, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 110230716928, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.58.ffn_norm.weight", "offset": 110771388928, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 110771409408, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 110774686208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 12 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 110783533568, "shape": [ 5120, 3072 ], "size": 8847360, "type": 11 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 110790291968, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 110797050368, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 110797052416, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.59.attn_kv_b.weight", "offset": 110798319616, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.59.attn_output.weight", "offset": 110805528576, "shape": [ 16384, 5120 ], "size": 7208960, "type": 12 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 110852714496, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 110852720640, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.59.attn_q_b.weight", "offset": 110856099840, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.59.attn_norm.weight", "offset": 110872320000, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 110872340480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 12 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 111580129280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 11 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 112120801280, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.59.ffn_norm.weight", "offset": 112661473280, "shape": [ 5120 ], "size": 540672000, "type": 0 } ], "version": 3 }