deepseek-coder-v2

An open-source Mixture-of-Experts code language model that achieves performance comparable to GPT4-Turbo in code-specific tasks.

16B 236B

111.6K Pulls Updated 12 days ago

deepseek-coder-v2:236b-instruct-q6_K ... /

model

c87d01d5cb6c · 194GB

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 18, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 14 }, { "name": "blk.0.attn_norm.weight", "offset": 860160000, "shape": [ 5120 ], "size": 430080000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 860180480, "shape": [ 12288, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 911790080, "shape": [ 5120, 12288 ], "size": 51609600, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 963399680, "shape": [ 5120, 12288 ], "size": 51609600, "type": 14 }, { "name": "blk.0.ffn_norm.weight", "offset": 1015009280, "shape": [ 5120 ], "size": 51609600, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 1015029760, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 1015031808, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.0.attn_kv_b.weight", "offset": 1017451008, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 1031213568, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 1100026368, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 1100032512, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.0.attn_q_b.weight", "offset": 1106483712, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1137449472, "shape": [ 5120, 160 ], "size": 30965760, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 1140726272, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 1153628672, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 1166531072, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 1179433472, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 1179435520, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.1.attn_kv_b.weight", "offset": 1181854720, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 1195617280, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 1264430080, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1264436224, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.1.attn_q_b.weight", "offset": 1270887424, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "output_norm.weight", "offset": 1301853184, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1301873664, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1301894144, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 2334086144, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 3366278144, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 4398470144, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 4398490624, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 4401767424, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 4414669824, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 4427572224, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 4440474624, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 4440476672, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.2.attn_kv_b.weight", "offset": 4442895872, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 4456658432, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 4525471232, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 4525477376, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.2.attn_q_b.weight", "offset": 4531928576, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 4562894336, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 4562914816, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 5595106816, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 6627298816, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 7659490816, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 7659511296, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 7662788096, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 7675690496, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 7688592896, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 7701495296, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 7701497344, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.3.attn_kv_b.weight", "offset": 7703916544, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 7717679104, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 7786491904, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 7786498048, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.3.attn_q_b.weight", "offset": 7792949248, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 7823915008, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 7823935488, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 8856127488, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 9888319488, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 10920511488, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 10920531968, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 10923808768, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 10936711168, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 10949613568, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 10962515968, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 10962518016, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.4.attn_kv_b.weight", "offset": 10964937216, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 10978699776, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 11047512576, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 11047518720, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.4.attn_q_b.weight", "offset": 11053969920, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 11084935680, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 11084956160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 12117148160, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 13149340160, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 14181532160, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 14181552640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 14181573120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 15213765120, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 16245957120, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 17278149120, "shape": [ 5120, 160 ], "size": 1032192000, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 17281425920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 17294328320, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 17307230720, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 17320133120, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 17320153600, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 17320155648, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.5.attn_kv_b.weight", "offset": 17322574848, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 17336337408, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 17405150208, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 17405156352, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.5.attn_q_b.weight", "offset": 17411607552, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 17442573312, "shape": [ 5120, 160 ], "size": 30965760, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 17445850112, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 17458752512, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 17471654912, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 17484557312, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 17484559360, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_kv_b.weight", "offset": 17486978560, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 17500741120, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 17569553920, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 17569560064, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.6.attn_q_b.weight", "offset": 17576011264, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 17606977024, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 17606997504, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 18639189504, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 19671381504, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 20703573504, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 20703593984, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 20706870784, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 20719773184, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 20732675584, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 20745577984, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 20745580032, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.7.attn_kv_b.weight", "offset": 20747999232, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 20761761792, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 20830574592, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 20830580736, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.7.attn_q_b.weight", "offset": 20837031936, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 20867997696, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 20868018176, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 21900210176, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 22932402176, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 23964594176, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 23964614656, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 23967891456, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 23980793856, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 23993696256, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 24006598656, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 24006600704, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.8.attn_kv_b.weight", "offset": 24009019904, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 24022782464, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 24091595264, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 24091601408, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.8.attn_q_b.weight", "offset": 24098052608, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 24129018368, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 24129038848, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 25161230848, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 26193422848, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 27225614848, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 27225635328, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 27228912128, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 27241814528, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 27254716928, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 27267619328, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 27267621376, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.9.attn_kv_b.weight", "offset": 27270040576, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 27283803136, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 27352615936, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 27352622080, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.9.attn_q_b.weight", "offset": 27359073280, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 27390039040, "shape": [ 5120, 160 ], "size": 30965760, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 27393315840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 27406218240, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 27419120640, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 27432023040, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 27432025088, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.10.attn_kv_b.weight", "offset": 27434444288, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 27448206848, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 27517019648, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 27517025792, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.10.attn_q_b.weight", "offset": 27523476992, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 27554442752, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 27554463232, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 28586655232, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 29618847232, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.9.ffn_norm.weight", "offset": 30651039232, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 30651059712, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 30651080192, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 31683272192, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 32715464192, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 33747656192, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 33747676672, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 33750953472, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 33763855872, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 33776758272, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 33789660672, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 33789662720, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.11.attn_kv_b.weight", "offset": 33792081920, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 33805844480, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 33874657280, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 33874663424, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.11.attn_q_b.weight", "offset": 33881114624, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 33912080384, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 33912100864, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 34944292864, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 35976484864, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 37008676864, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 37008697344, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 37011974144, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 37024876544, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 37037778944, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 37050681344, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 37050683392, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.12.attn_kv_b.weight", "offset": 37053102592, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 37066865152, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 37135677952, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 37135684096, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.12.attn_q_b.weight", "offset": 37142135296, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 37173101056, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 37173121536, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 38205313536, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 39237505536, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 40269697536, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 40269718016, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 40272994816, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 40285897216, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 40298799616, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 40311702016, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 40311704064, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.13.attn_kv_b.weight", "offset": 40314123264, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 40327885824, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 40396698624, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 40396704768, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.13.attn_q_b.weight", "offset": 40403155968, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 40434121728, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 40434142208, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 41466334208, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 42498526208, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 43530718208, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 43530738688, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 43534015488, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 43546917888, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 43559820288, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 43572722688, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 43572724736, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.14.attn_kv_b.weight", "offset": 43575143936, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 43588906496, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 43657719296, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 43657725440, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.14.attn_q_b.weight", "offset": 43664176640, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 43695142400, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 43695162880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 44727354880, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 45759546880, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 46791738880, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 46791759360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 46795036160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 46807938560, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 46820840960, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 46833743360, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 46833745408, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.15.attn_kv_b.weight", "offset": 46836164608, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 46849927168, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 46918739968, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 46918746112, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.15.attn_q_b.weight", "offset": 46925197312, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 46956163072, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 46956183552, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 47988375552, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 49020567552, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 50052759552, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 50052780032, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 50056056832, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 50068959232, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 50081861632, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 50094764032, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 50094766080, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.16.attn_kv_b.weight", "offset": 50097185280, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 50110947840, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 50179760640, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 50179766784, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.16.attn_q_b.weight", "offset": 50186217984, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 50217183744, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 50217204224, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 51249396224, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 52281588224, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 53313780224, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 53313800704, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 53317077504, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 53329979904, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 53342882304, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 53355784704, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 53355786752, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.17.attn_kv_b.weight", "offset": 53358205952, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 53371968512, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 53440781312, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 53440787456, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.17.attn_q_b.weight", "offset": 53447238656, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 53478204416, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 53478224896, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 54510416896, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 55542608896, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 56574800896, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 56574821376, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 56574841856, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 57607033856, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 58639225856, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 59671417856, "shape": [ 5120, 160 ], "size": 1032192000, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 59674694656, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 59687597056, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 59700499456, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 59713401856, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 59713422336, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 59713424384, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.18.attn_kv_b.weight", "offset": 59715843584, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 59729606144, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 59798418944, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 59798425088, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.18.attn_q_b.weight", "offset": 59804876288, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 59835842048, "shape": [ 5120, 160 ], "size": 30965760, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 59839118848, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 59852021248, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 59864923648, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 59877826048, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 59877828096, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.19.attn_kv_b.weight", "offset": 59880247296, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 59894009856, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 59962822656, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 59962828800, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.19.attn_q_b.weight", "offset": 59969280000, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 60000245760, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 60000266240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 61032458240, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 62064650240, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 63096842240, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 63096862720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 63100139520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 63113041920, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 63125944320, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 63138846720, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 63138848768, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.20.attn_kv_b.weight", "offset": 63141267968, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 63155030528, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 63223843328, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 63223849472, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.20.attn_q_b.weight", "offset": 63230300672, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 63261266432, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 63261286912, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 64293478912, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 65325670912, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 66357862912, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 66357883392, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 66361160192, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 66374062592, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 66386964992, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 66399867392, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "\u000e\u0000\u0000\u0000\u0000\ufffd\ufffdu\u000f\u0000\u0000\u0000\u0017\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.21.", "offset": 66399869440, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.21.attn_kv_b.weight", "offset": 66402288640, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 66416051200, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 66484864000, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 66484870144, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.21.attn_q_b.weight", "offset": 66491321344, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 66522287104, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 66522307584, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 67554499584, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 68586691584, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 69618883584, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 69618904064, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 69622180864, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 69635083264, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 69647985664, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 69660888064, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 69660890112, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.22.attn_kv_b.weight", "offset": 69663309312, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 69677071872, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 69745884672, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 69745890816, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.22.attn_q_b.weight", "offset": 69752342016, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 69783307776, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 69783328256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 70815520256, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 71847712256, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 72879904256, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 72879924736, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 72883201536, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 72896103936, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 72909006336, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 72921908736, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 72921910784, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.23.attn_kv_b.weight", "offset": 72924329984, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 72938092544, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 73006905344, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 73006911488, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.23.attn_q_b.weight", "offset": 73013362688, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 73044328448, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 73044348928, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 74076540928, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 75108732928, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 76140924928, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 76140945408, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 76144222208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 76157124608, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 76170027008, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 76182929408, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 76182931456, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.24.attn_kv_b.weight", "offset": 76185350656, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 76199113216, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 76267926016, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 76267932160, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.24.attn_q_b.weight", "offset": 76274383360, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 76305349120, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 76305369600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 77337561600, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 78369753600, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 79401945600, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 79401966080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 79405242880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 79418145280, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 79431047680, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 79443950080, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 79443952128, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.25.attn_kv_b.weight", "offset": 79446371328, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 79460133888, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 79528946688, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 79528952832, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.25.attn_q_b.weight", "offset": 79535404032, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 79566369792, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 79566390272, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 80598582272, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 81630774272, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 82662966272, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 82662986752, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 82666263552, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 82679165952, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 82692068352, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 82704970752, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 82704972800, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.26.attn_kv_b.weight", "offset": 82707392000, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 82721154560, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 82789967360, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 82789973504, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.26.attn_q_b.weight", "offset": 82796424704, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 82827390464, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 82827410944, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 83859602944, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 84891794944, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 85923986944, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 85924007424, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 85927284224, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 85940186624, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 85953089024, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 85965991424, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 85965993472, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.27.attn_kv_b.weight", "offset": 85968412672, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 85982175232, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 86050988032, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 86050994176, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.27.attn_q_b.weight", "offset": 86057445376, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 86088411136, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 86088431616, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 87120623616, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 88152815616, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.27.ffn_norm.weight", "offset": 89185007616, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 89185028096, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 89188304896, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 89201207296, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 89214109696, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 89227012096, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 89227014144, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.28.attn_kv_b.weight", "offset": 89229433344, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 89243195904, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 89312008704, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 89312014848, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.28.attn_q_b.weight", "offset": 89318466048, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 89349431808, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 89349452288, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 90381644288, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 91413836288, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.28.ffn_norm.weight", "offset": 92446028288, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 92446048768, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 92449325568, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 92462227968, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.29.ffn_up_shexp.weight", "offset": 92475130368, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 92488032768, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 92488034816, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.29.attn_kv_b.weight", "offset": 92490454016, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.29.attn_output.weight", "offset": 92504216576, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 92573029376, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 92573035520, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.29.attn_q_b.weight", "offset": 92579486720, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 92610452480, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 92610472960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 93642664960, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 94674856960, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.29.ffn_norm.weight", "offset": 95707048960, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 95707069440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 95710346240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 95723248640, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 95736151040, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 95749053440, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 95749055488, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.30.attn_kv_b.weight", "offset": 95751474688, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.30.attn_output.weight", "offset": 95765237248, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 95834050048, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 95834056192, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.30.attn_q_b.weight", "offset": 95840507392, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 95871473152, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 95871493632, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 96903685632, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 97935877632, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.30.ffn_norm.weight", "offset": 98968069632, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 98968090112, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 98968110592, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 100000302592, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 101032494592, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 102064686592, "shape": [ 5120, 160 ], "size": 1032192000, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 102067963392, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 102080865792, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 102093768192, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.31.ffn_norm.weight", "offset": 102106670592, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 102106691072, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 102106693120, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.31.attn_kv_b.weight", "offset": 102109112320, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.31.attn_output.weight", "offset": 102122874880, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 102191687680, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 102191693824, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.31.attn_q_b.weight", "offset": 102198145024, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 102229110784, "shape": [ 5120, 160 ], "size": 30965760, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 102232387584, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 102245289984, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 102258192384, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 102271094784, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 102271096832, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.32.attn_kv_b.weight", "offset": 102273516032, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.32.attn_output.weight", "offset": 102287278592, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 102356091392, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 102356097536, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.32.attn_q_b.weight", "offset": 102362548736, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 102393514496, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 102393534976, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 103425726976, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 104457918976, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.32.ffn_norm.weight", "offset": 105490110976, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 105490131456, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 105493408256, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 105506310656, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 105519213056, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 105532115456, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 105532117504, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.33.attn_kv_b.weight", "offset": 105534536704, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.33.attn_output.weight", "offset": 105548299264, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 105617112064, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 105617118208, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.33.attn_q_b.weight", "offset": 105623569408, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 105654535168, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 105654555648, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 106686747648, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 107718939648, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.33.ffn_norm.weight", "offset": 108751131648, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 108751152128, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 108754428928, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 108767331328, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 108780233728, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 108793136128, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 108793138176, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.34.attn_kv_b.weight", "offset": 108795557376, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.34.attn_output.weight", "offset": 108809319936, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 108878132736, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 108878138880, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.34.attn_q_b.weight", "offset": 108884590080, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 108915555840, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 108915576320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 109947768320, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 110979960320, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.34.ffn_norm.weight", "offset": 112012152320, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 112012172800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 112015449600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 112028352000, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 112041254400, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 112054156800, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 112054158848, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.35.attn_kv_b.weight", "offset": 112056578048, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 112070340608, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 112139153408, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 112139159552, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.35.attn_q_b.weight", "offset": 112145610752, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 112176576512, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 112176596992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 113208788992, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 114240980992, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.35.ffn_norm.weight", "offset": 115273172992, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 115273193472, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 115276470272, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 115289372672, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 115302275072, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 115315177472, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 115315179520, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.36.attn_kv_b.weight", "offset": 115317598720, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.36.attn_output.weight", "offset": 115331361280, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 115400174080, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 115400180224, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.36.attn_q_b.weight", "offset": 115406631424, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.36.attn_norm.weight", "offset": 115437597184, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 115437617664, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 116469809664, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 117502001664, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.36.ffn_norm.weight", "offset": 118534193664, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 118534214144, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 118537490944, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 118550393344, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 118563295744, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 118576198144, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 118576200192, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.37.attn_kv_b.weight", "offset": 118578619392, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.37.attn_output.weight", "offset": 118592381952, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 118661194752, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 118661200896, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.37.attn_q_b.weight", "offset": 118667652096, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 118698617856, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 118698638336, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 119730830336, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 120763022336, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.37.ffn_norm.weight", "offset": 121795214336, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 121795234816, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 121798511616, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 121811414016, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 121824316416, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 121837218816, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 121837220864, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.38.attn_kv_b.weight", "offset": 121839640064, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.38.attn_output.weight", "offset": 121853402624, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 121922215424, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 121922221568, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.38.attn_q_b.weight", "offset": 121928672768, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 121959638528, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 121959659008, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 122991851008, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 124024043008, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.38.ffn_norm.weight", "offset": 125056235008, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 125056255488, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 125059532288, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 125072434688, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 125085337088, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 125098239488, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 125098241536, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.39.attn_kv_b.weight", "offset": 125100660736, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.39.attn_output.weight", "offset": 125114423296, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 125183236096, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 125183242240, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.39.attn_q_b.weight", "offset": 125189693440, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 125220659200, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 125220679680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 126252871680, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 127285063680, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.39.ffn_norm.weight", "offset": 128317255680, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 128317276160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 128320552960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 128333455360, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 128346357760, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 128359260160, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 128359262208, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.40.attn_kv_b.weight", "offset": 128361681408, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.40.attn_output.weight", "offset": 128375443968, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 128444256768, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 128444262912, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.40.attn_q_b.weight", "offset": 128450714112, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 128481679872, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 128481700352, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 129513892352, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 130546084352, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.40.ffn_norm.weight", "offset": 131578276352, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 131578296832, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 131581573632, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 131594476032, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 131607378432, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 131620280832, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 131620282880, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.41.attn_kv_b.weight", "offset": 131622702080, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.41.attn_output.weight", "offset": 131636464640, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 131705277440, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 131705283584, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.41.attn_q_b.weight", "offset": 131711734784, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 131742700544, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 131742721024, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 132774913024, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 133807105024, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.41.ffn_norm.weight", "offset": 134839297024, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 134839317504, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 134842594304, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 134855496704, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 134868399104, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 134881301504, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 134881303552, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.42.attn_kv_b.weight", "offset": 134883722752, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.42.attn_output.weight", "offset": 134897485312, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 134966298112, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 134966304256, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.42.attn_q_b.weight", "offset": 134972755456, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.42.attn_norm.weight", "offset": 135003721216, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 135003741696, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 136035933696, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 137068125696, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.42.ffn_norm.weight", "offset": 138100317696, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 138100338176, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 138103614976, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 138116517376, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 138129419776, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 138142322176, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 138142324224, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.43.attn_kv_b.weight", "offset": 138144743424, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.43.attn_output.weight", "offset": 138158505984, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 138227318784, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 138227324928, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.43.attn_q_b.weight", "offset": 138233776128, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.43.attn_norm.weight", "offset": 138264741888, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 138264762368, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 139296954368, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 140329146368, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.43.ffn_norm.weight", "offset": 141361338368, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 141361358848, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 141361379328, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 142393571328, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 143425763328, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 144457955328, "shape": [ 5120, 160 ], "size": 1032192000, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 144461232128, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 144474134528, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 144487036928, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.44.ffn_norm.weight", "offset": 144499939328, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 144499959808, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 144499961856, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.44.attn_kv_b.weight", "offset": 144502381056, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.44.attn_output.weight", "offset": 144516143616, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 144584956416, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 144584962560, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.44.attn_q_b.weight", "offset": 144591413760, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 144622379520, "shape": [ 5120, 160 ], "size": 30965760, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 144625656320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 144638558720, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 144651461120, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 144664363520, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 144664365568, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.45.attn_kv_b.weight", "offset": 144666784768, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.45.attn_output.weight", "offset": 144680547328, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 144749360128, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 144749366272, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.45.attn_q_b.weight", "offset": 144755817472, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.45.attn_norm.weight", "offset": 144786783232, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 144786803712, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 145818995712, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 146851187712, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.45.ffn_norm.weight", "offset": 147883379712, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 147883400192, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 147886676992, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 147899579392, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 147912481792, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 147925384192, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 147925386240, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.46.attn_kv_b.weight", "offset": 147927805440, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.46.attn_output.weight", "offset": 147941568000, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 148010380800, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 148010386944, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.46.attn_q_b.weight", "offset": 148016838144, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.46.attn_norm.weight", "offset": 148047803904, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 148047824384, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 149080016384, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 150112208384, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.46.ffn_norm.weight", "offset": 151144400384, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 151144420864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 151147697664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 151160600064, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 151173502464, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 151186404864, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 151186406912, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.47.attn_kv_b.weight", "offset": 151188826112, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.47.attn_output.weight", "offset": 151202588672, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 151271401472, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 151271407616, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.47.attn_q_b.weight", "offset": 151277858816, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.47.attn_norm.weight", "offset": 151308824576, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 151308845056, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 152341037056, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 153373229056, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.47.ffn_norm.weight", "offset": 154405421056, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 154405441536, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 154408718336, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 154421620736, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 154434523136, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 154447425536, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 154447427584, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.48.attn_kv_b.weight", "offset": 154449846784, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.48.attn_output.weight", "offset": 154463609344, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 154532422144, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 154532428288, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.48.attn_q_b.weight", "offset": 154538879488, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.48.attn_norm.weight", "offset": 154569845248, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 154569865728, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 155602057728, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 156634249728, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.48.ffn_norm.weight", "offset": 157666441728, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 157666462208, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 157669739008, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 157682641408, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 157695543808, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 157708446208, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 157708448256, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.49.attn_kv_b.weight", "offset": 157710867456, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.49.attn_output.weight", "offset": 157724630016, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 157793442816, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 157793448960, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.49.attn_q_b.weight", "offset": 157799900160, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.49.attn_norm.weight", "offset": 157830865920, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 157830886400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 158863078400, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 159895270400, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.49.ffn_norm.weight", "offset": 160927462400, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 160927482880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 160930759680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 160943662080, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 160956564480, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 160969466880, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 160969468928, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.50.attn_kv_b.weight", "offset": 160971888128, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.50.attn_output.weight", "offset": 160985650688, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 161054463488, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 161054469632, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.50.attn_q_b.weight", "offset": 161060920832, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.50.attn_norm.weight", "offset": 161091886592, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 161091907072, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 162124099072, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 163156291072, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.50.ffn_norm.weight", "offset": 164188483072, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 164188503552, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 164191780352, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 164204682752, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 164217585152, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 164230487552, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 164230489600, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.51.attn_kv_b.weight", "offset": 164232908800, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.51.attn_output.weight", "offset": 164246671360, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 164315484160, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 164315490304, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.51.attn_q_b.weight", "offset": 164321941504, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.51.attn_norm.weight", "offset": 164352907264, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 164352927744, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 165385119744, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 166417311744, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.51.ffn_norm.weight", "offset": 167449503744, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 167449524224, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 167452801024, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 167465703424, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 167478605824, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 167491508224, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 167491510272, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.52.attn_kv_b.weight", "offset": 167493929472, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.52.attn_output.weight", "offset": 167507692032, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 167576504832, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 167576510976, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.52.attn_q_b.weight", "offset": 167582962176, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.52.attn_norm.weight", "offset": 167613927936, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 167613948416, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 168646140416, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 169678332416, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.52.ffn_norm.weight", "offset": 170710524416, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 170710544896, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 170713821696, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 170726724096, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 170739626496, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 170752528896, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 170752530944, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.53.attn_kv_b.weight", "offset": 170754950144, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.53.attn_output.weight", "offset": 170768712704, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 170837525504, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 170837531648, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.53.attn_q_b.weight", "offset": 170843982848, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.53.attn_norm.weight", "offset": 170874948608, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 170874969088, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 171907161088, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 172939353088, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.53.ffn_norm.weight", "offset": 173971545088, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 173971565568, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 173974842368, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 173987744768, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 174000647168, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 174013549568, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 174013551616, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.54.attn_kv_b.weight", "offset": 174015970816, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.54.attn_output.weight", "offset": 174029733376, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 174098546176, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 174098552320, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.54.attn_q_b.weight", "offset": 174105003520, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.54.attn_norm.weight", "offset": 174135969280, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 174135989760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 175168181760, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 176200373760, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.54.ffn_norm.weight", "offset": 177232565760, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 177232586240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 177235863040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 177248765440, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 177261667840, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 177274570240, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 177274572288, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.55.attn_kv_b.weight", "offset": 177276991488, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.55.attn_output.weight", "offset": 177290754048, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 177359566848, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 177359572992, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.55.attn_q_b.weight", "offset": 177366024192, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.55.attn_norm.weight", "offset": 177396989952, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 177397010432, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 178429202432, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 179461394432, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.55.ffn_norm.weight", "offset": 180493586432, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 180493606912, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 180496883712, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 180509786112, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 180522688512, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 180535590912, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 180535592960, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.56.attn_kv_b.weight", "offset": 180538012160, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.56.attn_output.weight", "offset": 180551774720, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 180620587520, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 180620593664, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.56.attn_q_b.weight", "offset": 180627044864, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.56.attn_norm.weight", "offset": 180658010624, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 180658031104, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 181690223104, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 182722415104, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.56.ffn_norm.weight", "offset": 183754607104, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 183754627584, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 183754648064, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 184786840064, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 185819032064, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 186851224064, "shape": [ 5120, 160 ], "size": 1032192000, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 186854500864, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 186867403264, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 186880305664, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.57.ffn_norm.weight", "offset": 186893208064, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 186893228544, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 186893230592, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.57.attn_kv_b.weight", "offset": 186895649792, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.57.attn_output.weight", "offset": 186909412352, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 186978225152, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 186978231296, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.57.attn_q_b.weight", "offset": 186984682496, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 187015648256, "shape": [ 5120, 160 ], "size": 30965760, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 187018925056, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 187031827456, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 187044729856, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 187057632256, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 187057634304, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.58.attn_kv_b.weight", "offset": 187060053504, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.58.attn_output.weight", "offset": 187073816064, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 187142628864, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 187142635008, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.58.attn_q_b.weight", "offset": 187149086208, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.58.attn_norm.weight", "offset": 187180051968, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 187180072448, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 188212264448, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 189244456448, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.58.ffn_norm.weight", "offset": 190276648448, "shape": [ 5120 ], "size": 1032192000, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 190276668928, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 190279945728, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 190292848128, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 190305750528, "shape": [ 5120, 3072 ], "size": 12902400, "type": 14 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 190318652928, "shape": [ 512 ], "size": 12902400, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 190318654976, "shape": [ 5120, 576 ], "size": 2048, "type": 14 }, { "name": "blk.59.attn_kv_b.weight", "offset": 190321074176, "shape": [ 512, 32768 ], "size": 2419200, "type": 14 }, { "name": "blk.59.attn_output.weight", "offset": 190334836736, "shape": [ 16384, 5120 ], "size": 13762560, "type": 14 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 190403649536, "shape": [ 1536 ], "size": 68812800, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 190403655680, "shape": [ 5120, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.59.attn_q_b.weight", "offset": 190410106880, "shape": [ 1536, 24576 ], "size": 6451200, "type": 14 }, { "name": "blk.59.attn_norm.weight", "offset": 190441072640, "shape": [ 5120 ], "size": 30965760, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 190441093120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 191473285120, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 192505477120, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 14 }, { "name": "blk.59.ffn_norm.weight", "offset": 193537669120, "shape": [ 5120 ], "size": 1032192000, "type": 0 } ], "version": 3 }