deepseek-coder-v2

An open-source Mixture-of-Experts code language model that achieves performance comparable to GPT4-Turbo in code-specific tasks.

16B 236B

111.6K Pulls Updated 12 days ago

deepseek-coder-v2:236b-instruct-q4_0 ... /

model

6bbfda8eb96d · 133GB

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 2, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 2 }, { "name": "blk.0.attn_norm.weight", "offset": 724992000, "shape": [ 5120 ], "size": 294912000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 725012480, "shape": [ 12288, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 760401920, "shape": [ 5120, 12288 ], "size": 35389440, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 795791360, "shape": [ 5120, 12288 ], "size": 35389440, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 831180800, "shape": [ 5120 ], "size": 35389440, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 831201280, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 831203328, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.0.attn_kv_b.weight", "offset": 832862208, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 842299392, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 889485312, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 889491456, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.0.attn_q_b.weight", "offset": 893915136, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 915148800, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 918425600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 927272960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 936120320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 944967680, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 944969728, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.1.attn_kv_b.weight", "offset": 946628608, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 956065792, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 1003251712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1003257856, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.1.attn_q_b.weight", "offset": 1007681536, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "output_norm.weight", "offset": 1028915200, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1028935680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1028956160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1736744960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2444533760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 3152322560, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3152343040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 3155619840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 3164467200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 3173314560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 3182161920, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 3182163968, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.2.attn_kv_b.weight", "offset": 3183822848, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 3193260032, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 3240445952, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 3240452096, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.2.attn_q_b.weight", "offset": 3244875776, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 3266109440, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3266129920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 3973918720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 4681707520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 5389496320, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 5389516800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 5392793600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 5401640960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 5410488320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 5419335680, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 5419337728, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.3.attn_kv_b.weight", "offset": 5420996608, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 5430433792, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 5477619712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 5477625856, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.3.attn_q_b.weight", "offset": 5482049536, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 5503283200, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 5503303680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 6211092480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 6918881280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 7626670080, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 7626690560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 7629967360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 7638814720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 7647662080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 7656509440, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 7656511488, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.4.attn_kv_b.weight", "offset": 7658170368, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 7667607552, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 7714793472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 7714799616, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.4.attn_q_b.weight", "offset": 7719223296, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 7740456960, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 7740477440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 8448266240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 9156055040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 9863843840, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 9863864320, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 9863884800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 10571673600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 11279462400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 11987251200, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 11990528000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 11999375360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 12008222720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 12017070080, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 12017090560, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 12017092608, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.5.attn_kv_b.weight", "offset": 12018751488, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 12028188672, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 12075374592, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 12075380736, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.5.attn_q_b.weight", "offset": 12079804416, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 12101038080, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 12104314880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 12113162240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 12122009600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 12130856960, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 12130859008, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.6.attn_kv_b.weight", "offset": 12132517888, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 12141955072, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 12189140992, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 12189147136, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.6.attn_q_b.weight", "offset": 12193570816, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 12214804480, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 12214824960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 12922613760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 13630402560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 14338191360, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 14338211840, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 14341488640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 14350336000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 14359183360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 14368030720, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 14368032768, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.7.attn_kv_b.weight", "offset": 14369691648, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 14379128832, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 14426314752, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 14426320896, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.7.attn_q_b.weight", "offset": 14430744576, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 14451978240, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 14451998720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 15159787520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 15867576320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 16575365120, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 16575385600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 16578662400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 16587509760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 16596357120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 16605204480, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 16605206528, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.8.attn_kv_b.weight", "offset": 16606865408, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 16616302592, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 16663488512, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 16663494656, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.8.attn_q_b.weight", "offset": 16667918336, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 16689152000, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 16689172480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 17396961280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 18104750080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 18812538880, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 18812559360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 18815836160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 18824683520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 18833530880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 18842378240, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 18842380288, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.9.attn_kv_b.weight", "offset": 18844039168, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 18853476352, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 18900662272, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 18900668416, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.9.attn_q_b.weight", "offset": 18905092096, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 18926325760, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 18929602560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 18938449920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 18947297280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 18956144640, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 18956146688, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.10.attn_kv_b.weight", "offset": 18957805568, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 18967242752, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 19014428672, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 19014434816, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.10.attn_q_b.weight", "offset": 19018858496, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 19040092160, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 19040112640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 19747901440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 20455690240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 21163479040, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 21163499520, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 21163520000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 21871308800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 22579097600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 23286886400, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 23286906880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 23290183680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 23299031040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 23307878400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 23316725760, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 23316727808, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.11.attn_kv_b.weight", "offset": 23318386688, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 23327823872, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 23375009792, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 23375015936, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.11.attn_q_b.weight", "offset": 23379439616, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 23400673280, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 23400693760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 24108482560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 24816271360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 25524060160, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 25524080640, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 25527357440, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 25536204800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 25545052160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 25553899520, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 25553901568, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.12.attn_kv_b.weight", "offset": 25555560448, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 25564997632, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 25612183552, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 25612189696, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.12.attn_q_b.weight", "offset": 25616613376, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 25637847040, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 25637867520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 26345656320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 27053445120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 27761233920, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 27761254400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 27764531200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 27773378560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 27782225920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 27791073280, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 27791075328, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.13.attn_kv_b.weight", "offset": 27792734208, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 27802171392, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 27849357312, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 27849363456, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.13.attn_q_b.weight", "offset": 27853787136, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 27875020800, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 27875041280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 28582830080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 29290618880, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 29998407680, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 29998428160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 30001704960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 30010552320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 30019399680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 30028247040, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 30028249088, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.14.attn_kv_b.weight", "offset": 30029907968, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 30039345152, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 30086531072, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 30086537216, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.14.attn_q_b.weight", "offset": 30090960896, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 30112194560, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 30112215040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 30820003840, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 31527792640, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 32235581440, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 32235601920, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 32238878720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 32247726080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 32256573440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 32265420800, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 32265422848, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.15.attn_kv_b.weight", "offset": 32267081728, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 32276518912, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 32323704832, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 32323710976, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.15.attn_q_b.weight", "offset": 32328134656, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 32349368320, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 32349388800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 33057177600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 33764966400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 34472755200, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 34472775680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 34476052480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 34484899840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 34493747200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 34502594560, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 34502596608, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.16.attn_kv_b.weight", "offset": 34504255488, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 34513692672, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 34560878592, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 34560884736, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.16.attn_q_b.weight", "offset": 34565308416, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 34586542080, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 34586562560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 35294351360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 36002140160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 36709928960, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 36709949440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 36713226240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 36722073600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 36730920960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 36739768320, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 36739770368, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.17.attn_kv_b.weight", "offset": 36741429248, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 36750866432, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 36798052352, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 36798058496, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.17.attn_q_b.weight", "offset": 36802482176, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 36823715840, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 36823736320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 37531525120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 38239313920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 38947102720, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 38947123200, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 38947143680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 39654932480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 40362721280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 41070510080, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 41073786880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 41082634240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 41091481600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 41100328960, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 41100349440, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 41100351488, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.18.attn_kv_b.weight", "offset": 41102010368, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 41111447552, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 41158633472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 41158639616, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.18.attn_q_b.weight", "offset": 41163063296, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 41184296960, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 41187573760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 41196421120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 41205268480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 41214115840, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 41214117888, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.19.attn_kv_b.weight", "offset": 41215776768, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 41225213952, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 41272399872, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 41272406016, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.19.attn_q_b.weight", "offset": 41276829696, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 41298063360, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 41298083840, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 42005872640, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 42713661440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 43421450240, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 43421470720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 43424747520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 43433594880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 43442442240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 43451289600, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 43451291648, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.20.attn_kv_b.weight", "offset": 43452950528, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 43462387712, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 43509573632, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 43509579776, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.20.attn_q_b.weight", "offset": 43514003456, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 43535237120, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 43535257600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 44243046400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 44950835200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 45658624000, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 45658644480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 45661921280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 45670768640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 45679616000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 45688463360, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "\u0002\u0000\u0000\u0000\u0000\ufffd\u003e\ufffd\n\u0000\u0000\u0000\u0017\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.21.", "offset": 45688465408, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.21.attn_kv_b.weight", "offset": 45690124288, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 45699561472, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 45746747392, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 45746753536, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.21.attn_q_b.weight", "offset": 45751177216, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 45772410880, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 45772431360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 46480220160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 47188008960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 47895797760, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 47895818240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 47899095040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 47907942400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 47916789760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 47925637120, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 47925639168, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.22.attn_kv_b.weight", "offset": 47927298048, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 47936735232, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 47983921152, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 47983927296, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.22.attn_q_b.weight", "offset": 47988350976, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 48009584640, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 48009605120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 48717393920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 49425182720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 50132971520, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 50132992000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 50136268800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 50145116160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 50153963520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 50162810880, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 50162812928, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.23.attn_kv_b.weight", "offset": 50164471808, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 50173908992, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 50221094912, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 50221101056, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.23.attn_q_b.weight", "offset": 50225524736, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 50246758400, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 50246778880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 50954567680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 51662356480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 52370145280, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 52370165760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 52373442560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 52382289920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 52391137280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 52399984640, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 52399986688, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.24.attn_kv_b.weight", "offset": 52401645568, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 52411082752, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 52458268672, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 52458274816, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.24.attn_q_b.weight", "offset": 52462698496, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 52483932160, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 52483952640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 53191741440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 53899530240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 54607319040, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 54607339520, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 54610616320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 54619463680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 54628311040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 54637158400, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 54637160448, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.25.attn_kv_b.weight", "offset": 54638819328, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 54648256512, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 54695442432, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 54695448576, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.25.attn_q_b.weight", "offset": 54699872256, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 54721105920, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 54721126400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 55428915200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 56136704000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 56844492800, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 56844513280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 56847790080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 56856637440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 56865484800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 56874332160, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 56874334208, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.26.attn_kv_b.weight", "offset": 56875993088, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 56885430272, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 56932616192, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 56932622336, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.26.attn_q_b.weight", "offset": 56937046016, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 56958279680, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 56958300160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 57666088960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 58373877760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 59081666560, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 59081687040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 59084963840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 59093811200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 59102658560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 59111505920, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 59111507968, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.27.attn_kv_b.weight", "offset": 59113166848, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 59122604032, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 59169789952, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 59169796096, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.27.attn_q_b.weight", "offset": 59174219776, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 59195453440, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 59195473920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 59903262720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 60611051520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 61318840320, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 61318860800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 61322137600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 61330984960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 61339832320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 61348679680, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 61348681728, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.28.attn_kv_b.weight", "offset": 61350340608, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 61359777792, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 61406963712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 61406969856, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.28.attn_q_b.weight", "offset": 61411393536, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 61432627200, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 61432647680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 62140436480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 62848225280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 63556014080, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 63556034560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 63559311360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 63568158720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.29.ffn_up_shexp.weight", "offset": 63577006080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 63585853440, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 63585855488, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.29.attn_kv_b.weight", "offset": 63587514368, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 63596951552, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 63644137472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 63644143616, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.29.attn_q_b.weight", "offset": 63648567296, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 63669800960, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 63669821440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 64377610240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 65085399040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 65793187840, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 65793208320, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 65796485120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 65805332480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 65814179840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 65823027200, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 65823029248, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.30.attn_kv_b.weight", "offset": 65824688128, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 65834125312, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 65881311232, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 65881317376, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.30.attn_q_b.weight", "offset": 65885741056, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 65906974720, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 65906995200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 66614784000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 67322572800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 68030361600, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 68030382080, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 68030402560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 68738191360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 69445980160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 70153768960, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 70157045760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 70165893120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 70174740480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 70183587840, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 70183608320, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 70183610368, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.31.attn_kv_b.weight", "offset": 70185269248, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.31.attn_output.weight", "offset": 70194706432, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 70241892352, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 70241898496, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.31.attn_q_b.weight", "offset": 70246322176, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 70267555840, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 70270832640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 70279680000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 70288527360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 70297374720, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 70297376768, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.32.attn_kv_b.weight", "offset": 70299035648, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.32.attn_output.weight", "offset": 70308472832, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 70355658752, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 70355664896, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.32.attn_q_b.weight", "offset": 70360088576, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 70381322240, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 70381342720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 71089131520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 71796920320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.32.ffn_norm.weight", "offset": 72504709120, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 72504729600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 72508006400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 72516853760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 72525701120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 72534548480, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 72534550528, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.33.attn_kv_b.weight", "offset": 72536209408, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.33.attn_output.weight", "offset": 72545646592, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 72592832512, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 72592838656, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.33.attn_q_b.weight", "offset": 72597262336, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.33.attn_norm.weight", "offset": 72618496000, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 72618516480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 73326305280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 74034094080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.33.ffn_norm.weight", "offset": 74741882880, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 74741903360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 74745180160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 74754027520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 74762874880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 74771722240, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 74771724288, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.34.attn_kv_b.weight", "offset": 74773383168, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.34.attn_output.weight", "offset": 74782820352, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 74830006272, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 74830012416, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.34.attn_q_b.weight", "offset": 74834436096, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 74855669760, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 74855690240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 75563479040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 76271267840, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.34.ffn_norm.weight", "offset": 76979056640, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 76979077120, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 76982353920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 76991201280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 77000048640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 77008896000, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 77008898048, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.35.attn_kv_b.weight", "offset": 77010556928, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.35.attn_output.weight", "offset": 77019994112, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 77067180032, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 77067186176, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.35.attn_q_b.weight", "offset": 77071609856, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.35.attn_norm.weight", "offset": 77092843520, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 77092864000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 77800652800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 78508441600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.35.ffn_norm.weight", "offset": 79216230400, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 79216250880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 79219527680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 79228375040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 79237222400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 79246069760, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 79246071808, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.36.attn_kv_b.weight", "offset": 79247730688, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.36.attn_output.weight", "offset": 79257167872, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 79304353792, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 79304359936, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.36.attn_q_b.weight", "offset": 79308783616, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.36.attn_norm.weight", "offset": 79330017280, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 79330037760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 80037826560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 80745615360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.36.ffn_norm.weight", "offset": 81453404160, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 81453424640, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 81456701440, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 81465548800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 81474396160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 81483243520, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 81483245568, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.37.attn_kv_b.weight", "offset": 81484904448, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.37.attn_output.weight", "offset": 81494341632, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 81541527552, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 81541533696, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.37.attn_q_b.weight", "offset": 81545957376, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.37.attn_norm.weight", "offset": 81567191040, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 81567211520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 82275000320, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 82982789120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.37.ffn_norm.weight", "offset": 83690577920, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 83690598400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 83693875200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 83702722560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 83711569920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 83720417280, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 83720419328, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.38.attn_kv_b.weight", "offset": 83722078208, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.38.attn_output.weight", "offset": 83731515392, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 83778701312, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 83778707456, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.38.attn_q_b.weight", "offset": 83783131136, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.38.attn_norm.weight", "offset": 83804364800, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 83804385280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 84512174080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 85219962880, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.38.ffn_norm.weight", "offset": 85927751680, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 85927772160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 85931048960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 85939896320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 85948743680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 85957591040, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 85957593088, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.39.attn_kv_b.weight", "offset": 85959251968, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.39.attn_output.weight", "offset": 85968689152, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 86015875072, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 86015881216, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.39.attn_q_b.weight", "offset": 86020304896, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 86041538560, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 86041559040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 86749347840, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 87457136640, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.39.ffn_norm.weight", "offset": 88164925440, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 88164945920, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 88168222720, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 88177070080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 88185917440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 88194764800, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 88194766848, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.40.attn_kv_b.weight", "offset": 88196425728, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.40.attn_output.weight", "offset": 88205862912, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 88253048832, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 88253054976, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.40.attn_q_b.weight", "offset": 88257478656, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.40.attn_norm.weight", "offset": 88278712320, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 88278732800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 88986521600, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 89694310400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.40.ffn_norm.weight", "offset": 90402099200, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 90402119680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 90405396480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 90414243840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 90423091200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 90431938560, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 90431940608, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.41.attn_kv_b.weight", "offset": 90433599488, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.41.attn_output.weight", "offset": 90443036672, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 90490222592, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 90490228736, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.41.attn_q_b.weight", "offset": 90494652416, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.41.attn_norm.weight", "offset": 90515886080, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 90515906560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 91223695360, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 91931484160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.41.ffn_norm.weight", "offset": 92639272960, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 92639293440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 92642570240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 92651417600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 92660264960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 92669112320, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 92669114368, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.42.attn_kv_b.weight", "offset": 92670773248, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.42.attn_output.weight", "offset": 92680210432, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 92727396352, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 92727402496, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.42.attn_q_b.weight", "offset": 92731826176, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.42.attn_norm.weight", "offset": 92753059840, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 92753080320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 93460869120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 94168657920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.42.ffn_norm.weight", "offset": 94876446720, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 94876467200, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 94879744000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 94888591360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 94897438720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 94906286080, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 94906288128, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.43.attn_kv_b.weight", "offset": 94907947008, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.43.attn_output.weight", "offset": 94917384192, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 94964570112, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 94964576256, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.43.attn_q_b.weight", "offset": 94968999936, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.43.attn_norm.weight", "offset": 94990233600, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 94990254080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 95698042880, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 96405831680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.43.ffn_norm.weight", "offset": 97113620480, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 97113640960, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 97113661440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 97821450240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 98529239040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 99237027840, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 99240304640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 99249152000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 99257999360, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.44.ffn_norm.weight", "offset": 99266846720, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 99266867200, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 99266869248, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.44.attn_kv_b.weight", "offset": 99268528128, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.44.attn_output.weight", "offset": 99277965312, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 99325151232, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 99325157376, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.44.attn_q_b.weight", "offset": 99329581056, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 99350814720, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 99354091520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 99362938880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 99371786240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 99380633600, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 99380635648, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.45.attn_kv_b.weight", "offset": 99382294528, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.45.attn_output.weight", "offset": 99391731712, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 99438917632, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 99438923776, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.45.attn_q_b.weight", "offset": 99443347456, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.45.attn_norm.weight", "offset": 99464581120, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 99464601600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 100172390400, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 100880179200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.45.ffn_norm.weight", "offset": 101587968000, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 101587988480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 101591265280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 101600112640, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 101608960000, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 101617807360, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 101617809408, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.46.attn_kv_b.weight", "offset": 101619468288, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.46.attn_output.weight", "offset": 101628905472, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 101676091392, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 101676097536, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.46.attn_q_b.weight", "offset": 101680521216, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.46.attn_norm.weight", "offset": 101701754880, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 101701775360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 102409564160, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 103117352960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.46.ffn_norm.weight", "offset": 103825141760, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 103825162240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 103828439040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 103837286400, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 103846133760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 103854981120, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 103854983168, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.47.attn_kv_b.weight", "offset": 103856642048, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.47.attn_output.weight", "offset": 103866079232, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 103913265152, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 103913271296, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.47.attn_q_b.weight", "offset": 103917694976, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.47.attn_norm.weight", "offset": 103938928640, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 103938949120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 104646737920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 105354526720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.47.ffn_norm.weight", "offset": 106062315520, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 106062336000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 106065612800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 106074460160, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 106083307520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 106092154880, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 106092156928, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.48.attn_kv_b.weight", "offset": 106093815808, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.48.attn_output.weight", "offset": 106103252992, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 106150438912, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 106150445056, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.48.attn_q_b.weight", "offset": 106154868736, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.48.attn_norm.weight", "offset": 106176102400, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 106176122880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 106883911680, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 107591700480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.48.ffn_norm.weight", "offset": 108299489280, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 108299509760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 108302786560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 108311633920, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 108320481280, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 108329328640, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 108329330688, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.49.attn_kv_b.weight", "offset": 108330989568, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.49.attn_output.weight", "offset": 108340426752, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 108387612672, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 108387618816, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.49.attn_q_b.weight", "offset": 108392042496, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.49.attn_norm.weight", "offset": 108413276160, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 108413296640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 109121085440, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 109828874240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.49.ffn_norm.weight", "offset": 110536663040, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 110536683520, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 110539960320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 110548807680, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 110557655040, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 110566502400, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 110566504448, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.50.attn_kv_b.weight", "offset": 110568163328, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.50.attn_output.weight", "offset": 110577600512, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 110624786432, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 110624792576, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.50.attn_q_b.weight", "offset": 110629216256, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.50.attn_norm.weight", "offset": 110650449920, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 110650470400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 111358259200, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 112066048000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.50.ffn_norm.weight", "offset": 112773836800, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 112773857280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 112777134080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 112785981440, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 112794828800, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 112803676160, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 112803678208, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.51.attn_kv_b.weight", "offset": 112805337088, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.51.attn_output.weight", "offset": 112814774272, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 112861960192, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 112861966336, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.51.attn_q_b.weight", "offset": 112866390016, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.51.attn_norm.weight", "offset": 112887623680, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 112887644160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 113595432960, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 114303221760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.51.ffn_norm.weight", "offset": 115011010560, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 115011031040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 115014307840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 115023155200, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 115032002560, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 115040849920, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 115040851968, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.52.attn_kv_b.weight", "offset": 115042510848, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.52.attn_output.weight", "offset": 115051948032, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 115099133952, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 115099140096, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.52.attn_q_b.weight", "offset": 115103563776, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.52.attn_norm.weight", "offset": 115124797440, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 115124817920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 115832606720, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 116540395520, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.52.ffn_norm.weight", "offset": 117248184320, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 117248204800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 117251481600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 117260328960, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 117269176320, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 117278023680, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 117278025728, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.53.attn_kv_b.weight", "offset": 117279684608, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.53.attn_output.weight", "offset": 117289121792, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 117336307712, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 117336313856, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.53.attn_q_b.weight", "offset": 117340737536, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.53.attn_norm.weight", "offset": 117361971200, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 117361991680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 118069780480, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 118777569280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.53.ffn_norm.weight", "offset": 119485358080, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 119485378560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 119488655360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 119497502720, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 119506350080, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 119515197440, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 119515199488, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.54.attn_kv_b.weight", "offset": 119516858368, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.54.attn_output.weight", "offset": 119526295552, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 119573481472, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 119573487616, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.54.attn_q_b.weight", "offset": 119577911296, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.54.attn_norm.weight", "offset": 119599144960, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 119599165440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 120306954240, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 121014743040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.54.ffn_norm.weight", "offset": 121722531840, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 121722552320, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 121725829120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 121734676480, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 121743523840, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 121752371200, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 121752373248, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.55.attn_kv_b.weight", "offset": 121754032128, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.55.attn_output.weight", "offset": 121763469312, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 121810655232, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 121810661376, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.55.attn_q_b.weight", "offset": 121815085056, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.55.attn_norm.weight", "offset": 121836318720, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 121836339200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 122544128000, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 123251916800, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.55.ffn_norm.weight", "offset": 123959705600, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 123959726080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 123963002880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 123971850240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 123980697600, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 123989544960, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 123989547008, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.56.attn_kv_b.weight", "offset": 123991205888, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.56.attn_output.weight", "offset": 124000643072, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 124047828992, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 124047835136, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.56.attn_q_b.weight", "offset": 124052258816, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.56.attn_norm.weight", "offset": 124073492480, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 124073512960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 124781301760, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 125489090560, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.56.ffn_norm.weight", "offset": 126196879360, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 126196899840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 126196920320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 126904709120, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 127612497920, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 128320286720, "shape": [ 5120, 160 ], "size": 707788800, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 128323563520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 128332410880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 128341258240, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.57.ffn_norm.weight", "offset": 128350105600, "shape": [ 5120 ], "size": 8847360, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 128350126080, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 128350128128, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.57.attn_kv_b.weight", "offset": 128351787008, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.57.attn_output.weight", "offset": 128361224192, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 128408410112, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 128408416256, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.57.attn_q_b.weight", "offset": 128412839936, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 128434073600, "shape": [ 5120, 160 ], "size": 21233664, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 128437350400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 128446197760, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 128455045120, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 128463892480, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 128463894528, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.58.attn_kv_b.weight", "offset": 128465553408, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.58.attn_output.weight", "offset": 128474990592, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 128522176512, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 128522182656, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.58.attn_q_b.weight", "offset": 128526606336, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.58.attn_norm.weight", "offset": 128547840000, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 128547860480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 129255649280, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 129963438080, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.58.ffn_norm.weight", "offset": 130671226880, "shape": [ 5120 ], "size": 707788800, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 130671247360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 130674524160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 2 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 130683371520, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 130692218880, "shape": [ 5120, 3072 ], "size": 8847360, "type": 2 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 130701066240, "shape": [ 512 ], "size": 8847360, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 130701068288, "shape": [ 5120, 576 ], "size": 2048, "type": 2 }, { "name": "blk.59.attn_kv_b.weight", "offset": 130702727168, "shape": [ 512, 32768 ], "size": 1658880, "type": 2 }, { "name": "blk.59.attn_output.weight", "offset": 130712164352, "shape": [ 16384, 5120 ], "size": 9437184, "type": 2 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 130759350272, "shape": [ 1536 ], "size": 47185920, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 130759356416, "shape": [ 5120, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.59.attn_q_b.weight", "offset": 130763780096, "shape": [ 1536, 24576 ], "size": 4423680, "type": 2 }, { "name": "blk.59.attn_norm.weight", "offset": 130785013760, "shape": [ 5120 ], "size": 21233664, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 130785034240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 2 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 131492823040, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 132200611840, "shape": [ 5120, 1536, 160 ], "size": 707788800, "type": 2 }, { "name": "blk.59.ffn_norm.weight", "offset": 132908400640, "shape": [ 5120 ], "size": 707788800, "type": 0 } ], "version": 3 }