deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 10, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 10 }, { "name": "blk.0.attn_norm.weight", "offset": 602112000, "shape": [ 5120 ], "size": 172032000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 602132480, "shape": [ 12288, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 629166080, "shape": [ 5120, 12288 ], "size": 27033600, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 649809920, "shape": [ 5120, 12288 ], "size": 20643840, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 670453760, "shape": [ 5120 ], "size": 20643840, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 670474240, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 670476288, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.0.attn_kv_b.weight", "offset": 671443968, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 676948992, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 712993792, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 712999936, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.0.attn_q_b.weight", "offset": 715580416, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 727966720, "shape": [ 5120, 160 ], "size": 12386304, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 731243520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 738001920, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 743162880, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 748323840, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 748325888, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.1.attn_kv_b.weight", "offset": 749293568, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 754798592, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 790843392, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 790849536, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.1.attn_q_b.weight", "offset": 793430016, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "output_norm.weight", "offset": 805816320, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 805836800, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 805857280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1346529280, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 1759406080, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 2172282880, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2172303360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 2175580160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 2182338560, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 2187499520, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 2192660480, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 2192662528, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.2.attn_kv_b.weight", "offset": 2193630208, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 2199135232, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 2235180032, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 2235186176, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.2.attn_q_b.weight", "offset": 2237766656, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.2.attn_norm.weight", "offset": 2250152960, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 2250173440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 2790845440, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 3203722240, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 3616599040, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 3616619520, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 3619896320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 3626654720, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 3631815680, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 3636976640, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 3636978688, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.3.attn_kv_b.weight", "offset": 3637946368, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 3643451392, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 3679496192, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 3679502336, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.3.attn_q_b.weight", "offset": 3682082816, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.3.attn_norm.weight", "offset": 3694469120, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 3694489600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 4235161600, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 4648038400, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 5060915200, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 5060935680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 5064212480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 5070970880, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 5076131840, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 5081292800, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 5081294848, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.4.attn_kv_b.weight", "offset": 5082262528, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 5087767552, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 5123812352, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 5123818496, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.4.attn_q_b.weight", "offset": 5126398976, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.4.attn_norm.weight", "offset": 5138785280, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 5138805760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 5679477760, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 6092354560, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 6505231360, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 6505251840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 6505272320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 7045944320, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 7458821120, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 7871697920, "shape": [ 5120, 160 ], "size": 412876800, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 7874974720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 7881733120, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 7886894080, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 7892055040, "shape": [ 5120 ], "size": 5160960, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 7892075520, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 7892077568, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.5.attn_kv_b.weight", "offset": 7893045248, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 7898550272, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 7934595072, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 7934601216, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.5.attn_q_b.weight", "offset": 7937181696, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 7949568000, "shape": [ 5120, 160 ], "size": 12386304, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 7952844800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 7959603200, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 7964764160, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 7969925120, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 7969927168, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.6.attn_kv_b.weight", "offset": 7970894848, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 7976399872, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 8012444672, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 8012450816, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.6.attn_q_b.weight", "offset": 8015031296, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.6.attn_norm.weight", "offset": 8027417600, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 8027438080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 8568110080, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 8980986880, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.6.ffn_norm.weight", "offset": 9393863680, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 9393884160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 9397160960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 9403919360, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 9409080320, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 9414241280, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 9414243328, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.7.attn_kv_b.weight", "offset": 9415211008, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 9420716032, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 9456760832, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 9456766976, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.7.attn_q_b.weight", "offset": 9459347456, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.7.attn_norm.weight", "offset": 9471733760, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 9471754240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 10012426240, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 10425303040, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 10838179840, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 10838200320, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 10841477120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 10848235520, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 10853396480, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 10858557440, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 10858559488, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.8.attn_kv_b.weight", "offset": 10859527168, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 10865032192, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 10901076992, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 10901083136, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.8.attn_q_b.weight", "offset": 10903663616, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.8.attn_norm.weight", "offset": 10916049920, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 10916070400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 11456742400, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 11869619200, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 12282496000, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 12282516480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 12285793280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 12292551680, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 12297712640, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 12302873600, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 12302875648, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.9.attn_kv_b.weight", "offset": 12303843328, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 12309348352, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 12345393152, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 12345399296, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.9.attn_q_b.weight", "offset": 12347979776, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 12360366080, "shape": [ 5120, 160 ], "size": 12386304, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 12363642880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 12370401280, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 12375562240, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 12380723200, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 12380725248, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.10.attn_kv_b.weight", "offset": 12381692928, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 12387197952, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 12423242752, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 12423248896, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.10.attn_q_b.weight", "offset": 12425829376, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.9.attn_norm.weight", "offset": 12438215680, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 12438236160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 12978908160, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 13391784960, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 13804661760, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 13804682240, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 13804702720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 14345374720, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 14758251520, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 15171128320, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 15171148800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 15174425600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 15181184000, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 15186344960, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 15191505920, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 15191507968, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.11.attn_kv_b.weight", "offset": 15192475648, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 15197980672, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 15234025472, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 15234031616, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.11.attn_q_b.weight", "offset": 15236612096, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.11.attn_norm.weight", "offset": 15248998400, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 15249018880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 15789690880, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 16202567680, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 16615444480, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 16615464960, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 16618741760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 16625500160, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 16630661120, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 16635822080, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 16635824128, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.12.attn_kv_b.weight", "offset": 16636791808, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 16642296832, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 16678341632, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 16678347776, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.12.attn_q_b.weight", "offset": 16680928256, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.12.attn_norm.weight", "offset": 16693314560, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 16693335040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 17234007040, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 17646883840, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 18059760640, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 18059781120, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 18063057920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 18069816320, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 18074977280, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 18080138240, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 18080140288, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.13.attn_kv_b.weight", "offset": 18081107968, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 18086612992, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 18122657792, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 18122663936, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.13.attn_q_b.weight", "offset": 18125244416, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.13.attn_norm.weight", "offset": 18137630720, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 18137651200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 18678323200, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 19091200000, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 19504076800, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 19504097280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 19507374080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 19514132480, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 19519293440, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 19524454400, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 19524456448, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.14.attn_kv_b.weight", "offset": 19525424128, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 19530929152, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 19566973952, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 19566980096, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.14.attn_q_b.weight", "offset": 19569560576, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.14.attn_norm.weight", "offset": 19581946880, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 19581967360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 20122639360, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 20535516160, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 20948392960, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 20948413440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 20951690240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 20958448640, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 20963609600, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 20968770560, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 20968772608, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.15.attn_kv_b.weight", "offset": 20969740288, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 20975245312, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 21011290112, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 21011296256, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.15.attn_q_b.weight", "offset": 21013876736, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.15.attn_norm.weight", "offset": 21026263040, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 21026283520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 21566955520, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 21979832320, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 22392709120, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 22392729600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 22396006400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 22402764800, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 22407925760, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 22413086720, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 22413088768, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.16.attn_kv_b.weight", "offset": 22414056448, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 22419561472, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 22455606272, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 22455612416, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.16.attn_q_b.weight", "offset": 22458192896, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.16.attn_norm.weight", "offset": 22470579200, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 22470599680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 23011271680, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 23424148480, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 23837025280, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 23837045760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 23840322560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 23847080960, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 23852241920, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 23857402880, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 23857404928, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.17.attn_kv_b.weight", "offset": 23858372608, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 23863877632, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 23899922432, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 23899928576, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.17.attn_q_b.weight", "offset": 23902509056, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.17.attn_norm.weight", "offset": 23914895360, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 23914915840, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 24455587840, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 24868464640, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 25281341440, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 25281361920, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 25281382400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 25822054400, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 26234931200, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 26647808000, "shape": [ 5120, 160 ], "size": 412876800, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 26651084800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 26657843200, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 26663004160, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 26668165120, "shape": [ 5120 ], "size": 5160960, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 26668185600, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 26668187648, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.18.attn_kv_b.weight", "offset": 26669155328, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 26674660352, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 26710705152, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 26710711296, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.18.attn_q_b.weight", "offset": 26713291776, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 26725678080, "shape": [ 5120, 160 ], "size": 12386304, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 26728954880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 26735713280, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 26740874240, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 26746035200, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 26746037248, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.19.attn_kv_b.weight", "offset": 26747004928, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 26752509952, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 26788554752, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 26788560896, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.19.attn_q_b.weight", "offset": 26791141376, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.19.attn_norm.weight", "offset": 26803527680, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 26803548160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 27344220160, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 27757096960, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 28169973760, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 28169994240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 28173271040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 28180029440, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 28185190400, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 28190351360, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 28190353408, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.20.attn_kv_b.weight", "offset": 28191321088, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 28196826112, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 28232870912, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 28232877056, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.20.attn_q_b.weight", "offset": 28235457536, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.20.attn_norm.weight", "offset": 28247843840, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 28247864320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 28788536320, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 29201413120, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 29614289920, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 29614310400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 29617587200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 29624345600, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 29629506560, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 29634667520, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "ight\u0001\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u00004\ufffd\u003e\b\u0000\u0000", "offset": 29634669568, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.21.attn_kv_b.weight", "offset": 29635637248, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 29641142272, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 29677187072, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 29677193216, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.21.attn_q_b.weight", "offset": 29679773696, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.21.attn_norm.weight", "offset": 29692160000, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 29692180480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 30232852480, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 30645729280, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 31058606080, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 31058626560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 31061903360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 31068661760, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 31073822720, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 31078983680, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 31078985728, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.22.attn_kv_b.weight", "offset": 31079953408, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 31085458432, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 31121503232, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 31121509376, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.22.attn_q_b.weight", "offset": 31124089856, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.22.attn_norm.weight", "offset": 31136476160, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 31136496640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 31677168640, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 32090045440, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 32502922240, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 32502942720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 32506219520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 32512977920, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 32518138880, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 32523299840, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 32523301888, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.23.attn_kv_b.weight", "offset": 32524269568, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 32529774592, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 32565819392, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 32565825536, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.23.attn_q_b.weight", "offset": 32568406016, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.23.attn_norm.weight", "offset": 32580792320, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 32580812800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 33121484800, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 33534361600, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 33947238400, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 33947258880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 33950535680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 33957294080, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 33962455040, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 33967616000, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 33967618048, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.24.attn_kv_b.weight", "offset": 33968585728, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 33974090752, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 34010135552, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 34010141696, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.24.attn_q_b.weight", "offset": 34012722176, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.24.attn_norm.weight", "offset": 34025108480, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 34025128960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 34565800960, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 34978677760, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 35391554560, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 35391575040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 35394851840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 35401610240, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 35406771200, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 35411932160, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 35411934208, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.25.attn_kv_b.weight", "offset": 35412901888, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 35418406912, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 35454451712, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 35454457856, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.25.attn_q_b.weight", "offset": 35457038336, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.25.attn_norm.weight", "offset": 35469424640, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 35469445120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 36010117120, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 36422993920, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 36835870720, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 36835891200, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 36839168000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 36845926400, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 36851087360, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 36856248320, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 36856250368, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.26.attn_kv_b.weight", "offset": 36857218048, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 36862723072, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 36898767872, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 36898774016, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.26.attn_q_b.weight", "offset": 36901354496, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.26.attn_norm.weight", "offset": 36913740800, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 36913761280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 37454433280, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 37867310080, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 38280186880, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 38280207360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 38283484160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 38290242560, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 38295403520, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 38300564480, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 38300566528, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.27.attn_kv_b.weight", "offset": 38301534208, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.27.attn_output.weight", "offset": 38307039232, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 38343084032, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 38343090176, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.27.attn_q_b.weight", "offset": 38345670656, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.27.attn_norm.weight", "offset": 38358056960, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 38358077440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 38898749440, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 39311626240, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.27.ffn_norm.weight", "offset": 39724503040, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 39724523520, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 39727800320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 39734558720, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 39739719680, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 39744880640, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 39744882688, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.28.attn_kv_b.weight", "offset": 39745850368, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 39751355392, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 39787400192, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 39787406336, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.28.attn_q_b.weight", "offset": 39789986816, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.28.attn_norm.weight", "offset": 39802373120, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 39802393600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 40343065600, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 40755942400, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.28.ffn_norm.weight", "offset": 41168819200, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 41168839680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 41172116480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 41178874880, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.29.ffn_up_shexp.weight", "offset": 41184035840, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 41189196800, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 41189198848, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.29.attn_kv_b.weight", "offset": 41190166528, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.29.attn_output.weight", "offset": 41195671552, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 41231716352, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 41231722496, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.29.attn_q_b.weight", "offset": 41234302976, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.29.attn_norm.weight", "offset": 41246689280, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 41246709760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 41787381760, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 42200258560, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.29.ffn_norm.weight", "offset": 42613135360, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 42613155840, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 42616432640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 42623191040, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 42628352000, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 42633512960, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 42633515008, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.30.attn_kv_b.weight", "offset": 42634482688, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.30.attn_output.weight", "offset": 42639987712, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 42676032512, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 42676038656, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.30.attn_q_b.weight", "offset": 42678619136, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.30.attn_norm.weight", "offset": 42691005440, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 42691025920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 43231697920, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 43644574720, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.30.ffn_norm.weight", "offset": 44057451520, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 44057472000, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 44057492480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 44598164480, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 45011041280, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 45423918080, "shape": [ 5120, 160 ], "size": 412876800, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 45427194880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 45433953280, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 45439114240, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.31.ffn_norm.weight", "offset": 45444275200, "shape": [ 5120 ], "size": 5160960, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 45444295680, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 45444297728, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.31.attn_kv_b.weight", "offset": 45445265408, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.31.attn_output.weight", "offset": 45450770432, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 45486815232, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 45486821376, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.31.attn_q_b.weight", "offset": 45489401856, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 45501788160, "shape": [ 5120, 160 ], "size": 12386304, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 45505064960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 45511823360, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 45516984320, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 45522145280, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 45522147328, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.32.attn_kv_b.weight", "offset": 45523115008, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.32.attn_output.weight", "offset": 45528620032, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 45564664832, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 45564670976, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.32.attn_q_b.weight", "offset": 45567251456, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.32.attn_norm.weight", "offset": 45579637760, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 45579658240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 46120330240, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 46533207040, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.32.ffn_norm.weight", "offset": 46946083840, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 46946104320, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 46949381120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 46956139520, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 46961300480, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 46966461440, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 46966463488, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.33.attn_kv_b.weight", "offset": 46967431168, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.33.attn_output.weight", "offset": 46972936192, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 47008980992, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 47008987136, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.33.attn_q_b.weight", "offset": 47011567616, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.33.attn_norm.weight", "offset": 47023953920, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 47023974400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 47564646400, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 47977523200, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.33.ffn_norm.weight", "offset": 48390400000, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 48390420480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 48393697280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 48400455680, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 48405616640, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 48410777600, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 48410779648, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.34.attn_kv_b.weight", "offset": 48411747328, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.34.attn_output.weight", "offset": 48417252352, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 48453297152, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 48453303296, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.34.attn_q_b.weight", "offset": 48455883776, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.34.attn_norm.weight", "offset": 48468270080, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 48468290560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 49008962560, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 49421839360, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.34.ffn_norm.weight", "offset": 49834716160, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 49834736640, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 49838013440, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 49844771840, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 49849932800, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 49855093760, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 49855095808, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.35.attn_kv_b.weight", "offset": 49856063488, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 49861568512, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 49897613312, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 49897619456, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.35.attn_q_b.weight", "offset": 49900199936, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.35.attn_norm.weight", "offset": 49912586240, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 49912606720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 50453278720, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 50866155520, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.35.ffn_norm.weight", "offset": 51279032320, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 51279052800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 51282329600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 51289088000, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 51294248960, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 51299409920, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 51299411968, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.36.attn_kv_b.weight", "offset": 51300379648, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.36.attn_output.weight", "offset": 51305884672, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 51341929472, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 51341935616, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.36.attn_q_b.weight", "offset": 51344516096, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.36.attn_norm.weight", "offset": 51356902400, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 51356922880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 51897594880, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 52310471680, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.36.ffn_norm.weight", "offset": 52723348480, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 52723368960, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 52726645760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 52733404160, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 52738565120, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 52743726080, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 52743728128, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.37.attn_kv_b.weight", "offset": 52744695808, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.37.attn_output.weight", "offset": 52750200832, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 52786245632, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 52786251776, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.37.attn_q_b.weight", "offset": 52788832256, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.37.attn_norm.weight", "offset": 52801218560, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 52801239040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 53341911040, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 53754787840, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.37.ffn_norm.weight", "offset": 54167664640, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 54167685120, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 54170961920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 54177720320, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 54182881280, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 54188042240, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 54188044288, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.38.attn_kv_b.weight", "offset": 54189011968, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.38.attn_output.weight", "offset": 54194516992, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 54230561792, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 54230567936, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.38.attn_q_b.weight", "offset": 54233148416, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.38.attn_norm.weight", "offset": 54245534720, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 54245555200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 54786227200, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 55199104000, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.38.ffn_norm.weight", "offset": 55611980800, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 55612001280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 55615278080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 55622036480, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 55627197440, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 55632358400, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 55632360448, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.39.attn_kv_b.weight", "offset": 55633328128, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.39.attn_output.weight", "offset": 55638833152, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 55674877952, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 55674884096, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.39.attn_q_b.weight", "offset": 55677464576, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.39.attn_norm.weight", "offset": 55689850880, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 55689871360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 56230543360, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 56643420160, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.39.ffn_norm.weight", "offset": 57056296960, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 57056317440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 57059594240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 57066352640, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 57071513600, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 57076674560, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 57076676608, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.40.attn_kv_b.weight", "offset": 57077644288, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.40.attn_output.weight", "offset": 57083149312, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 57119194112, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 57119200256, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.40.attn_q_b.weight", "offset": 57121780736, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.40.attn_norm.weight", "offset": 57134167040, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 57134187520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 57674859520, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 58087736320, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.40.ffn_norm.weight", "offset": 58500613120, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 58500633600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 58503910400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 58510668800, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 58515829760, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 58520990720, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 58520992768, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.41.attn_kv_b.weight", "offset": 58521960448, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.41.attn_output.weight", "offset": 58527465472, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 58563510272, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 58563516416, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.41.attn_q_b.weight", "offset": 58566096896, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.41.attn_norm.weight", "offset": 58578483200, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 58578503680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 59119175680, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 59532052480, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.41.ffn_norm.weight", "offset": 59944929280, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 59944949760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 59948226560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 59954984960, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 59960145920, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 59965306880, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 59965308928, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.42.attn_kv_b.weight", "offset": 59966276608, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.42.attn_output.weight", "offset": 59971781632, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 60007826432, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 60007832576, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.42.attn_q_b.weight", "offset": 60010413056, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.42.attn_norm.weight", "offset": 60022799360, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 60022819840, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 60563491840, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 60976368640, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.42.ffn_norm.weight", "offset": 61389245440, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 61389265920, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 61392542720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 61399301120, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 61404462080, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 61409623040, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 61409625088, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.43.attn_kv_b.weight", "offset": 61410592768, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.43.attn_output.weight", "offset": 61416097792, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 61452142592, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 61452148736, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.43.attn_q_b.weight", "offset": 61454729216, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.43.attn_norm.weight", "offset": 61467115520, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 61467136000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 62007808000, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 62420684800, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.43.ffn_norm.weight", "offset": 62833561600, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 62833582080, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 62833602560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 63374274560, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 63787151360, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 64200028160, "shape": [ 5120, 160 ], "size": 412876800, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 64203304960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 64210063360, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 64215224320, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.44.ffn_norm.weight", "offset": 64220385280, "shape": [ 5120 ], "size": 5160960, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 64220405760, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 64220407808, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.44.attn_kv_b.weight", "offset": 64221375488, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.44.attn_output.weight", "offset": 64226880512, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 64262925312, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 64262931456, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.44.attn_q_b.weight", "offset": 64265511936, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 64277898240, "shape": [ 5120, 160 ], "size": 12386304, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 64281175040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 64287933440, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 64293094400, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 64298255360, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 64298257408, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.45.attn_kv_b.weight", "offset": 64299225088, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.45.attn_output.weight", "offset": 64304730112, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 64340774912, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 64340781056, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.45.attn_q_b.weight", "offset": 64343361536, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.45.attn_norm.weight", "offset": 64355747840, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 64355768320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 64896440320, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 65309317120, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.45.ffn_norm.weight", "offset": 65722193920, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 65722214400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 65725491200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 65732249600, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 65737410560, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 65742571520, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 65742573568, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.46.attn_kv_b.weight", "offset": 65743541248, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.46.attn_output.weight", "offset": 65749046272, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 65785091072, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 65785097216, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.46.attn_q_b.weight", "offset": 65787677696, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.46.attn_norm.weight", "offset": 65800064000, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 65800084480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 66340756480, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 66753633280, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.46.ffn_norm.weight", "offset": 67166510080, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 67166530560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 67169807360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 67176565760, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 67181726720, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 67186887680, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 67186889728, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.47.attn_kv_b.weight", "offset": 67187857408, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.47.attn_output.weight", "offset": 67193362432, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 67229407232, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 67229413376, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.47.attn_q_b.weight", "offset": 67231993856, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.47.attn_norm.weight", "offset": 67244380160, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 67244400640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 67785072640, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 68197949440, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.47.ffn_norm.weight", "offset": 68610826240, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 68610846720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 68614123520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 68620881920, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 68626042880, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 68631203840, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 68631205888, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.48.attn_kv_b.weight", "offset": 68632173568, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.48.attn_output.weight", "offset": 68637678592, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 68673723392, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 68673729536, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.48.attn_q_b.weight", "offset": 68676310016, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.48.attn_norm.weight", "offset": 68688696320, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 68688716800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 69229388800, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 69642265600, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.48.ffn_norm.weight", "offset": 70055142400, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 70055162880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 70058439680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 70065198080, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 70070359040, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 70075520000, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 70075522048, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.49.attn_kv_b.weight", "offset": 70076489728, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.49.attn_output.weight", "offset": 70081994752, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 70118039552, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 70118045696, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.49.attn_q_b.weight", "offset": 70120626176, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.49.attn_norm.weight", "offset": 70133012480, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 70133032960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 70673704960, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 71086581760, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.49.ffn_norm.weight", "offset": 71499458560, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 71499479040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 71502755840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 71509514240, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 71514675200, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 71519836160, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 71519838208, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.50.attn_kv_b.weight", "offset": 71520805888, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.50.attn_output.weight", "offset": 71526310912, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 71562355712, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 71562361856, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.50.attn_q_b.weight", "offset": 71564942336, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.50.attn_norm.weight", "offset": 71577328640, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 71577349120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 72118021120, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 72530897920, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.50.ffn_norm.weight", "offset": 72943774720, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 72943795200, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 72947072000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 72953830400, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 72958991360, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 72964152320, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 72964154368, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.51.attn_kv_b.weight", "offset": 72965122048, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.51.attn_output.weight", "offset": 72970627072, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 73006671872, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 73006678016, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.51.attn_q_b.weight", "offset": 73009258496, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.51.attn_norm.weight", "offset": 73021644800, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 73021665280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 73562337280, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 73975214080, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.51.ffn_norm.weight", "offset": 74388090880, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 74388111360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 74391388160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 74398146560, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 74403307520, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 74408468480, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 74408470528, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.52.attn_kv_b.weight", "offset": 74409438208, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.52.attn_output.weight", "offset": 74414943232, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 74450988032, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 74450994176, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.52.attn_q_b.weight", "offset": 74453574656, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.52.attn_norm.weight", "offset": 74465960960, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 74465981440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 75006653440, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 75419530240, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.52.ffn_norm.weight", "offset": 75832407040, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 75832427520, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 75835704320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 75842462720, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 75847623680, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 75852784640, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 75852786688, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.53.attn_kv_b.weight", "offset": 75853754368, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.53.attn_output.weight", "offset": 75859259392, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 75895304192, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 75895310336, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.53.attn_q_b.weight", "offset": 75897890816, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.53.attn_norm.weight", "offset": 75910277120, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 75910297600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 76450969600, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 76863846400, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.53.ffn_norm.weight", "offset": 77276723200, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 77276743680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 77280020480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 77286778880, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 77291939840, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 77297100800, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 77297102848, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.54.attn_kv_b.weight", "offset": 77298070528, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.54.attn_output.weight", "offset": 77303575552, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 77339620352, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 77339626496, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.54.attn_q_b.weight", "offset": 77342206976, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.54.attn_norm.weight", "offset": 77354593280, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 77354613760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 77895285760, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 78308162560, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.54.ffn_norm.weight", "offset": 78721039360, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 78721059840, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 78724336640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 78731095040, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 78736256000, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 78741416960, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 78741419008, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.55.attn_kv_b.weight", "offset": 78742386688, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.55.attn_output.weight", "offset": 78747891712, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 78783936512, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 78783942656, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.55.attn_q_b.weight", "offset": 78786523136, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.55.attn_norm.weight", "offset": 78798909440, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 78798929920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 79339601920, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 79752478720, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.55.ffn_norm.weight", "offset": 80165355520, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 80165376000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 80168652800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 80175411200, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 80180572160, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 80185733120, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 80185735168, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.56.attn_kv_b.weight", "offset": 80186702848, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.56.attn_output.weight", "offset": 80192207872, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 80228252672, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 80228258816, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.56.attn_q_b.weight", "offset": 80230839296, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.56.attn_norm.weight", "offset": 80243225600, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 80243246080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 80783918080, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 81196794880, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.56.ffn_norm.weight", "offset": 81609671680, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 81609692160, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 81609712640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 82150384640, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 82563261440, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 82976138240, "shape": [ 5120, 160 ], "size": 412876800, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 82979415040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 82986173440, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 82991334400, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.57.ffn_norm.weight", "offset": 82996495360, "shape": [ 5120 ], "size": 5160960, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 82996515840, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 82996517888, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.57.attn_kv_b.weight", "offset": 82997485568, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.57.attn_output.weight", "offset": 83002990592, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 83039035392, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 83039041536, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.57.attn_q_b.weight", "offset": 83041622016, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 83054008320, "shape": [ 5120, 160 ], "size": 12386304, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 83057285120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 83064043520, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 83069204480, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 83074365440, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 83074367488, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.58.attn_kv_b.weight", "offset": 83075335168, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.58.attn_output.weight", "offset": 83080840192, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 83116884992, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 83116891136, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.58.attn_q_b.weight", "offset": 83119471616, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.58.attn_norm.weight", "offset": 83131857920, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 83131878400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 83672550400, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 84085427200, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.58.ffn_norm.weight", "offset": 84498304000, "shape": [ 5120 ], "size": 412876800, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 84498324480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 84501601280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 84508359680, "shape": [ 5120, 3072 ], "size": 6758400, "type": 10 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 84513520640, "shape": [ 5120, 3072 ], "size": 5160960, "type": 10 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 84518681600, "shape": [ 512 ], "size": 5160960, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 84518683648, "shape": [ 5120, 576 ], "size": 2048, "type": 10 }, { "name": "blk.59.attn_kv_b.weight", "offset": 84519651328, "shape": [ 512, 32768 ], "size": 967680, "type": 10 }, { "name": "blk.59.attn_output.weight", "offset": 84525156352, "shape": [ 16384, 5120 ], "size": 5505024, "type": 11 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 84561201152, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 84561207296, "shape": [ 5120, 1536 ], "size": 6144, "type": 10 }, { "name": "blk.59.attn_q_b.weight", "offset": 84563787776, "shape": [ 1536, 24576 ], "size": 2580480, "type": 10 }, { "name": "blk.59.attn_norm.weight", "offset": 84576174080, "shape": [ 5120 ], "size": 12386304, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 84576194560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 85116866560, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 10 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 85529743360, "shape": [ 5120, 1536, 160 ], "size": 412876800, "type": 10 }, { "name": "blk.59.ffn_norm.weight", "offset": 85942620160, "shape": [ 5120 ], "size": 412876800, "type": 0 } ], "version": 3 }