deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 11, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 655360000, "shape": [ 5120 ], "size": 225280000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 655380480, "shape": [ 12288, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 682414080, "shape": [ 5120, 12288 ], "size": 27033600, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 709447680, "shape": [ 5120, 12288 ], "size": 27033600, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 736481280, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 736501760, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 736503808, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_kv_b.weight", "offset": 737771008, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 744979968, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 781024768, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 781030912, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.attn_q_b.weight", "offset": 784410112, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 800630272, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 803907072, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 810665472, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 817423872, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 824182272, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 824184320, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_kv_b.weight", "offset": 825451520, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 832660480, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "\u0000\u0015\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.1.attn_q_b.we", "offset": 868705280, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 868711424, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.attn_q_b.weight", "offset": 872090624, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "output_norm.weight", "offset": 888310784, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 888331264, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 888351744, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 1429023744, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 1969695744, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 2510367744, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 2510388224, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 2513665024, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 2520423424, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 2527181824, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 2533940224, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 2533942272, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_kv_b.weight", "offset": 2535209472, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 2542418432, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 2578463232, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 2578469376, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.attn_q_b.weight", "offset": 2581848576, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 2598068736, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 2598089216, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 3138761216, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 3679433216, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 4220105216, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 4220125696, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 4223402496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 4230160896, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 4236919296, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 4243677696, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 4243679744, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_kv_b.weight", "offset": 4244946944, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 4252155904, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 4288200704, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 4288206848, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.attn_q_b.weight", "offset": 4291586048, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 4307806208, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 4307826688, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 4848498688, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 5389170688, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 5929842688, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 5929863168, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 5933139968, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 5939898368, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 5946656768, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 5953415168, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 5953417216, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_kv_b.weight", "offset": 5954684416, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 5961893376, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 5997938176, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 5997944320, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.attn_q_b.weight", "offset": 6001323520, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 6017543680, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 6017564160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 6558236160, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 7098908160, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 7639580160, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 7639600640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 7639621120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 8180293120, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 8720965120, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 9261637120, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 9264913920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 9271672320, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 9278430720, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 9285189120, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 9285209600, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 9285211648, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_kv_b.weight", "offset": 9286478848, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 9293687808, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 9329732608, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 9329738752, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.attn_q_b.weight", "offset": 9333117952, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 9349338112, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 9352614912, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 9359373312, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 9366131712, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 9372890112, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 9372892160, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_kv_b.weight", "offset": 9374159360, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 9381368320, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 9417413120, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 9417419264, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.attn_q_b.weight", "offset": 9420798464, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 9437018624, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 9437039104, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 9977711104, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 10518383104, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 11059055104, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 11059075584, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 11062352384, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 11069110784, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 11075869184, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 11082627584, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 11082629632, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_kv_b.weight", "offset": 11083896832, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 11091105792, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 11127150592, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 11127156736, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.attn_q_b.weight", "offset": 11130535936, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 11146756096, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 11146776576, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 11687448576, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 12228120576, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 12768792576, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 12768813056, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 12772089856, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 12778848256, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 12785606656, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 12792365056, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 12792367104, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_kv_b.weight", "offset": 12793634304, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 12800843264, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 12836888064, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 12836894208, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.attn_q_b.weight", "offset": 12840273408, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 12856493568, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 12856514048, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 13397186048, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 13937858048, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 14478530048, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 14478550528, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 14481827328, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 14488585728, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 14495344128, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 14502102528, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 14502104576, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_kv_b.weight", "offset": 14503371776, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 14510580736, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 14546625536, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 14546631680, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.attn_q_b.weight", "offset": 14550010880, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 14566231040, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 14569507840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 14576266240, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 14583024640, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 14589783040, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 14589785088, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_kv_b.weight", "offset": 14591052288, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 14598261248, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 14634306048, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 14634312192, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.attn_q_b.weight", "offset": 14637691392, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 14653911552, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 14653932032, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 15194604032, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 15735276032, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 16275948032, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 16275968512, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 16275988992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 16816660992, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 17357332992, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 17898004992, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 17898025472, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 17901302272, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 17908060672, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 17914819072, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 17921577472, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 17921579520, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_kv_b.weight", "offset": 17922846720, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 17930055680, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 17966100480, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 17966106624, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.11.attn_q_b.weight", "offset": 17969485824, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 17985705984, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 17985726464, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 18526398464, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 19067070464, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 19607742464, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 19607762944, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 19611039744, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 19617798144, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 19624556544, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 19631314944, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 19631316992, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_kv_b.weight", "offset": 19632584192, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 19639793152, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 19675837952, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 19675844096, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.attn_q_b.weight", "offset": 19679223296, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 19695443456, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 19695463936, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 20236135936, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 20776807936, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 21317479936, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 21317500416, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 21320777216, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 21327535616, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 21334294016, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 21341052416, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 21341054464, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_kv_b.weight", "offset": 21342321664, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 21349530624, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 21385575424, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 21385581568, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.attn_q_b.weight", "offset": 21388960768, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 21405180928, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 21405201408, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 21945873408, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 22486545408, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 23027217408, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 23027237888, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 23030514688, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 23037273088, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 23044031488, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 23050789888, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 23050791936, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_kv_b.weight", "offset": 23052059136, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 23059268096, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 23095312896, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 23095319040, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.attn_q_b.weight", "offset": 23098698240, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 23114918400, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 23114938880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 23655610880, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 24196282880, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 24736954880, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 24736975360, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 24740252160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 24747010560, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 24753768960, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 24760527360, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 24760529408, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_kv_b.weight", "offset": 24761796608, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 24769005568, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 24805050368, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 24805056512, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.attn_q_b.weight", "offset": 24808435712, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 24824655872, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 24824676352, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 25365348352, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 25906020352, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 26446692352, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 26446712832, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 26449989632, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 26456748032, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 26463506432, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 26470264832, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 26470266880, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_kv_b.weight", "offset": 26471534080, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 26478743040, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 26514787840, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 26514793984, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.attn_q_b.weight", "offset": 26518173184, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 26534393344, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 26534413824, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 27075085824, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 27615757824, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 28156429824, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 28156450304, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 28159727104, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 28166485504, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 28173243904, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 28180002304, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 28180004352, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_kv_b.weight", "offset": 28181271552, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 28188480512, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 28224525312, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 28224531456, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.attn_q_b.weight", "offset": 28227910656, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 28244130816, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 28244151296, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 28784823296, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 29325495296, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 29866167296, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 29866187776, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 29866208256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 30406880256, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 30947552256, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 31488224256, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 31491501056, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 31498259456, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 31505017856, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 31511776256, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 31511796736, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 31511798784, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_kv_b.weight", "offset": 31513065984, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 31520274944, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 31556319744, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 31556325888, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.attn_q_b.weight", "offset": 31559705088, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 31575925248, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 31579202048, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 31585960448, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 31592718848, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 31599477248, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 31599479296, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_kv_b.weight", "offset": 31600746496, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 31607955456, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 31644000256, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 31644006400, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.attn_q_b.weight", "offset": 31647385600, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 31663605760, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 31663626240, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 32204298240, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 32744970240, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 33285642240, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 33285662720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 33288939520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 33295697920, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 33302456320, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 33309214720, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 33309216768, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_kv_b.weight", "offset": 33310483968, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 33317692928, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 33353737728, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 33353743872, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.attn_q_b.weight", "offset": 33357123072, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 33373343232, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 33373363712, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 33914035712, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 34454707712, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 34995379712, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 34995400192, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 34998676992, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 35005435392, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 35012193792, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 35018952192, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "ight\u0001\u0000\u0000\u0000\u0000\u0002\u0000_kv_a_mqa.weight", "offset": 35018954240, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_kv_b.weight", "offset": 35020221440, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 35027430400, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 35063475200, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 35063481344, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.attn_q_b.weight", "offset": 35066860544, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 35083080704, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 35083101184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 35623773184, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 36164445184, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 36705117184, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 36705137664, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 36708414464, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 36715172864, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 36721931264, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 36728689664, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 36728691712, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_kv_b.weight", "offset": 36729958912, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 36737167872, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 36773212672, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 36773218816, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.attn_q_b.weight", "offset": 36776598016, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 36792818176, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 36792838656, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 37333510656, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 37874182656, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 38414854656, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 38414875136, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 38418151936, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 38424910336, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 38431668736, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 38438427136, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 38438429184, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_kv_b.weight", "offset": 38439696384, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 38446905344, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 38482950144, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 38482956288, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.attn_q_b.weight", "offset": 38486335488, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 38502555648, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 38502576128, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 39043248128, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 39583920128, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 40124592128, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 40124612608, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 40127889408, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 40134647808, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 40141406208, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 40148164608, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 40148166656, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_kv_b.weight", "offset": 40149433856, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 40156642816, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 40192687616, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 40192693760, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.attn_q_b.weight", "offset": 40196072960, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 40212293120, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 40212313600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 40752985600, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 41293657600, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 41834329600, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 41834350080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 41837626880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 41844385280, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 41851143680, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 41857902080, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 41857904128, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_kv_b.weight", "offset": 41859171328, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 41866380288, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 41902425088, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 41902431232, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.attn_q_b.weight", "offset": 41905810432, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 41922030592, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 41922051072, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 42462723072, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 43003395072, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 43544067072, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 43544087552, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 43547364352, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 43554122752, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 43560881152, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 43567639552, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 43567641600, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_kv_b.weight", "offset": 43568908800, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 43576117760, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 43612162560, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 43612168704, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.attn_q_b.weight", "offset": 43615547904, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 43631768064, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 43631788544, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 44172460544, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 44713132544, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 45253804544, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 45253825024, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 45257101824, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 45263860224, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 45270618624, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 45277377024, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 45277379072, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.27.attn_kv_b.weight", "offset": 45278646272, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 45285855232, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 45321900032, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 45321906176, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.attn_q_b.weight", "offset": 45325285376, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 45341505536, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 45341526016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 45882198016, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 46422870016, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 46963542016, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 46963562496, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 46966839296, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 46973597696, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 46980356096, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 46987114496, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 46987116544, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.28.attn_kv_b.weight", "offset": 46988383744, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 46995592704, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 47031637504, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 47031643648, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.28.attn_q_b.weight", "offset": 47035022848, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 47051243008, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 47051263488, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 47591935488, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 48132607488, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 48673279488, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 48673299968, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 48676576768, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 48683335168, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "hexp.weight\u0002\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000", "offset": 48690093568, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 48696851968, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 48696854016, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.29.attn_kv_b.weight", "offset": 48698121216, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 48705330176, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 48741374976, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 48741381120, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.29.attn_q_b.weight", "offset": 48744760320, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.29.attn_norm.weight", "offset": 48760980480, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 48761000960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 49301672960, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 49842344960, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 50383016960, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 50383037440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 50386314240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 50393072640, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 50399831040, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 50406589440, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 50406591488, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.30.attn_kv_b.weight", "offset": 50407858688, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 50415067648, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 50451112448, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 50451118592, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.30.attn_q_b.weight", "offset": 50454497792, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 50470717952, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 50470738432, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 51011410432, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 51552082432, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 52092754432, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 52092774912, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 52092795392, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 52633467392, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 53174139392, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 53714811392, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 53718088192, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 53724846592, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 53731604992, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 53738363392, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 53738383872, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 53738385920, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.31.attn_kv_b.weight", "offset": 53739653120, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.31.attn_output.weight", "offset": 53746862080, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 53782906880, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 53782913024, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.31.attn_q_b.weight", "offset": 53786292224, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 53802512384, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 53805789184, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 53812547584, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 53819305984, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 53826064384, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 53826066432, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.32.attn_kv_b.weight", "offset": 53827333632, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 53834542592, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 53870587392, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 53870593536, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.32.attn_q_b.weight", "offset": 53873972736, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 53890192896, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 53890213376, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 54430885376, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 54971557376, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 55512229376, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 55512249856, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 55515526656, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 55522285056, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 55529043456, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 55535801856, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 55535803904, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.33.attn_kv_b.weight", "offset": 55537071104, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 55544280064, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 55580324864, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 55580331008, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.33.attn_q_b.weight", "offset": 55583710208, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 55599930368, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 55599950848, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 56140622848, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 56681294848, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 57221966848, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 57221987328, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 57225264128, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 57232022528, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 57238780928, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 57245539328, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 57245541376, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.34.attn_kv_b.weight", "offset": 57246808576, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 57254017536, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 57290062336, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 57290068480, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.34.attn_q_b.weight", "offset": 57293447680, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 57309667840, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 57309688320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 57850360320, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 58391032320, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 58931704320, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 58931724800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 58935001600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 58941760000, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 58948518400, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 58955276800, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 58955278848, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.35.attn_kv_b.weight", "offset": 58956546048, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 58963755008, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 58999799808, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 58999805952, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.35.attn_q_b.weight", "offset": 59003185152, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.35.attn_norm.weight", "offset": 59019405312, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 59019425792, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 59560097792, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 60100769792, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 60641441792, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 60641462272, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 60644739072, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 60651497472, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 60658255872, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 60665014272, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 60665016320, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.36.attn_kv_b.weight", "offset": 60666283520, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 60673492480, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 60709537280, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 60709543424, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.36.attn_q_b.weight", "offset": 60712922624, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.36.attn_norm.weight", "offset": 60729142784, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 60729163264, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 61269835264, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 61810507264, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 62351179264, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 62351199744, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 62354476544, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 62361234944, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 62367993344, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 62374751744, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 62374753792, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.37.attn_kv_b.weight", "offset": 62376020992, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 62383229952, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 62419274752, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 62419280896, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.37.attn_q_b.weight", "offset": 62422660096, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 62438880256, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 62438900736, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 62979572736, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 63520244736, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 64060916736, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 64060937216, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 64064214016, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 64070972416, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 64077730816, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 64084489216, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 64084491264, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.38.attn_kv_b.weight", "offset": 64085758464, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 64092967424, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 64129012224, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 64129018368, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.38.attn_q_b.weight", "offset": 64132397568, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 64148617728, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 64148638208, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 64689310208, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 65229982208, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 65770654208, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 65770674688, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 65773951488, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 65780709888, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 65787468288, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 65794226688, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 65794228736, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.39.attn_kv_b.weight", "offset": 65795495936, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 65802704896, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 65838749696, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 65838755840, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.39.attn_q_b.weight", "offset": 65842135040, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 65858355200, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 65858375680, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 66399047680, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 66939719680, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 67480391680, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 67480412160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 67483688960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 67490447360, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 67497205760, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 67503964160, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 67503966208, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.40.attn_kv_b.weight", "offset": 67505233408, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 67512442368, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 67548487168, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 67548493312, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.40.attn_q_b.weight", "offset": 67551872512, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.40.attn_norm.weight", "offset": 67568092672, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 67568113152, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 68108785152, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 68649457152, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.40.ffn_norm.weight", "offset": 69190129152, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 69190149632, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 69193426432, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 69200184832, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 69206943232, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 69213701632, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 69213703680, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.41.attn_kv_b.weight", "offset": 69214970880, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 69222179840, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 69258224640, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 69258230784, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.41.attn_q_b.weight", "offset": 69261609984, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.41.attn_norm.weight", "offset": 69277830144, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 69277850624, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 69818522624, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 70359194624, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.41.ffn_norm.weight", "offset": 70899866624, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 70899887104, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 70903163904, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 70909922304, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 70916680704, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 70923439104, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 70923441152, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.42.attn_kv_b.weight", "offset": 70924708352, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.42.attn_output.weight", "offset": 70931917312, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 70967962112, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 70967968256, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.42.attn_q_b.weight", "offset": 70971347456, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.42.attn_norm.weight", "offset": 70987567616, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 70987588096, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 71528260096, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 72068932096, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.42.ffn_norm.weight", "offset": 72609604096, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 72609624576, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 72612901376, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 72619659776, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 72626418176, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 72633176576, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 72633178624, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.43.attn_kv_b.weight", "offset": 72634445824, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.43.attn_output.weight", "offset": 72641654784, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 72677699584, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 72677705728, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.43.attn_q_b.weight", "offset": 72681084928, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.43.attn_norm.weight", "offset": 72697305088, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 72697325568, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 73237997568, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 73778669568, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.43.ffn_norm.weight", "offset": 74319341568, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 74319362048, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 74319382528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 74860054528, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 75400726528, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 75941398528, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 75944675328, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 75951433728, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 75958192128, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.44.ffn_norm.weight", "offset": 75964950528, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 75964971008, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 75964973056, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.44.attn_kv_b.weight", "offset": 75966240256, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.44.attn_output.weight", "offset": 75973449216, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 76009494016, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 76009500160, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.44.attn_q_b.weight", "offset": 76012879360, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 76029099520, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 76032376320, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 76039134720, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 76045893120, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 76052651520, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 76052653568, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.45.attn_kv_b.weight", "offset": 76053920768, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.45.attn_output.weight", "offset": 76061129728, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 76097174528, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 76097180672, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.45.attn_q_b.weight", "offset": 76100559872, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.45.attn_norm.weight", "offset": 76116780032, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 76116800512, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 76657472512, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 77198144512, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.45.ffn_norm.weight", "offset": 77738816512, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 77738836992, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 77742113792, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 77748872192, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 77755630592, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 77762388992, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 77762391040, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.46.attn_kv_b.weight", "offset": 77763658240, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.46.attn_output.weight", "offset": 77770867200, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 77806912000, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 77806918144, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.46.attn_q_b.weight", "offset": 77810297344, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.46.attn_norm.weight", "offset": 77826517504, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 77826537984, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 78367209984, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 78907881984, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.46.ffn_norm.weight", "offset": 79448553984, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 79448574464, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 79451851264, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 79458609664, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 79465368064, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 79472126464, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 79472128512, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.47.attn_kv_b.weight", "offset": 79473395712, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.47.attn_output.weight", "offset": 79480604672, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 79516649472, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 79516655616, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.47.attn_q_b.weight", "offset": 79520034816, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.47.attn_norm.weight", "offset": 79536254976, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 79536275456, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 80076947456, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 80617619456, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.47.ffn_norm.weight", "offset": 81158291456, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 81158311936, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 81161588736, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 81168347136, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 81175105536, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 81181863936, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 81181865984, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.48.attn_kv_b.weight", "offset": 81183133184, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.48.attn_output.weight", "offset": 81190342144, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 81226386944, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 81226393088, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.48.attn_q_b.weight", "offset": 81229772288, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.48.attn_norm.weight", "offset": 81245992448, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 81246012928, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 81786684928, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 82327356928, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.48.ffn_norm.weight", "offset": 82868028928, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 82868049408, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 82871326208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 82878084608, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 82884843008, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 82891601408, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 82891603456, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.49.attn_kv_b.weight", "offset": 82892870656, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.49.attn_output.weight", "offset": 82900079616, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 82936124416, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 82936130560, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.49.attn_q_b.weight", "offset": 82939509760, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.49.attn_norm.weight", "offset": 82955729920, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 82955750400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 83496422400, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 84037094400, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.49.ffn_norm.weight", "offset": 84577766400, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 84577786880, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 84581063680, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 84587822080, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 84594580480, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 84601338880, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 84601340928, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.50.attn_kv_b.weight", "offset": 84602608128, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.50.attn_output.weight", "offset": 84609817088, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 84645861888, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 84645868032, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.50.attn_q_b.weight", "offset": 84649247232, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.50.attn_norm.weight", "offset": 84665467392, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 84665487872, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 85206159872, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 85746831872, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.50.ffn_norm.weight", "offset": 86287503872, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 86287524352, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 86290801152, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 86297559552, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 86304317952, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 86311076352, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 86311078400, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.51.attn_kv_b.weight", "offset": 86312345600, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.51.attn_output.weight", "offset": 86319554560, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 86355599360, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 86355605504, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.51.attn_q_b.weight", "offset": 86358984704, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.51.attn_norm.weight", "offset": 86375204864, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 86375225344, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 86915897344, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 87456569344, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.51.ffn_norm.weight", "offset": 87997241344, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 87997261824, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 88000538624, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 88007297024, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 88014055424, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 88020813824, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 88020815872, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.52.attn_kv_b.weight", "offset": 88022083072, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.52.attn_output.weight", "offset": 88029292032, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 88065336832, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 88065342976, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.52.attn_q_b.weight", "offset": 88068722176, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.52.attn_norm.weight", "offset": 88084942336, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 88084962816, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 88625634816, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 89166306816, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.52.ffn_norm.weight", "offset": 89706978816, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 89706999296, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 89710276096, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 89717034496, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 89723792896, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 89730551296, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 89730553344, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.53.attn_kv_b.weight", "offset": 89731820544, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.53.attn_output.weight", "offset": 89739029504, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 89775074304, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 89775080448, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.53.attn_q_b.weight", "offset": 89778459648, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.53.attn_norm.weight", "offset": 89794679808, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 89794700288, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 90335372288, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 90876044288, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.53.ffn_norm.weight", "offset": 91416716288, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 91416736768, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 91420013568, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 91426771968, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 91433530368, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 91440288768, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 91440290816, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.54.attn_kv_b.weight", "offset": 91441558016, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.54.attn_output.weight", "offset": 91448766976, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 91484811776, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 91484817920, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.54.attn_q_b.weight", "offset": 91488197120, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.54.attn_norm.weight", "offset": 91504417280, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 91504437760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 92045109760, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 92585781760, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.54.ffn_norm.weight", "offset": 93126453760, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 93126474240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 93129751040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 93136509440, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 93143267840, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 93150026240, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 93150028288, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.55.attn_kv_b.weight", "offset": 93151295488, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.55.attn_output.weight", "offset": 93158504448, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 93194549248, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 93194555392, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.55.attn_q_b.weight", "offset": 93197934592, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.55.attn_norm.weight", "offset": 93214154752, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 93214175232, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 93754847232, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 94295519232, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.55.ffn_norm.weight", "offset": 94836191232, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 94836211712, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 94839488512, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 94846246912, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 94853005312, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 94859763712, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 94859765760, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.56.attn_kv_b.weight", "offset": 94861032960, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.56.attn_output.weight", "offset": 94868241920, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 94904286720, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 94904292864, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.56.attn_q_b.weight", "offset": 94907672064, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.56.attn_norm.weight", "offset": 94923892224, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 94923912704, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 95464584704, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 96005256704, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.56.ffn_norm.weight", "offset": 96545928704, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 96545949184, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 96545969664, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 97086641664, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 97627313664, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 98167985664, "shape": [ 5120, 160 ], "size": 540672000, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 98171262464, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 98178020864, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 98184779264, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.57.ffn_norm.weight", "offset": 98191537664, "shape": [ 5120 ], "size": 6758400, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 98191558144, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 98191560192, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.57.attn_kv_b.weight", "offset": 98192827392, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.57.attn_output.weight", "offset": 98200036352, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 98236081152, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 98236087296, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.57.attn_q_b.weight", "offset": 98239466496, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 98255686656, "shape": [ 5120, 160 ], "size": 16220160, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 98258963456, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 98265721856, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 98272480256, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 98279238656, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 98279240704, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.58.attn_kv_b.weight", "offset": 98280507904, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.58.attn_output.weight", "offset": 98287716864, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 98323761664, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 98323767808, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.58.attn_q_b.weight", "offset": 98327147008, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.58.attn_norm.weight", "offset": 98343367168, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 98343387648, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 98884059648, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 99424731648, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.58.ffn_norm.weight", "offset": 99965403648, "shape": [ 5120 ], "size": 540672000, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 99965424128, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 99968700928, "shape": [ 3072, 5120 ], "size": 3276800, "type": 11 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 99975459328, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 99982217728, "shape": [ 5120, 3072 ], "size": 6758400, "type": 11 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 99988976128, "shape": [ 512 ], "size": 6758400, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 99988978176, "shape": [ 5120, 576 ], "size": 2048, "type": 11 }, { "name": "blk.59.attn_kv_b.weight", "offset": 99990245376, "shape": [ 512, 32768 ], "size": 1267200, "type": 11 }, { "name": "blk.59.attn_output.weight", "offset": 99997454336, "shape": [ 16384, 5120 ], "size": 7208960, "type": 11 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 100033499136, "shape": [ 1536 ], "size": 36044800, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 100033505280, "shape": [ 5120, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.59.attn_q_b.weight", "offset": 100036884480, "shape": [ 1536, 24576 ], "size": 3379200, "type": 11 }, { "name": "blk.59.attn_norm.weight", "offset": 100053104640, "shape": [ 5120 ], "size": 16220160, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 100053125120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 11 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 100593797120, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 101134469120, "shape": [ 5120, 1536, 160 ], "size": 540672000, "type": 11 }, { "name": "blk.59.ffn_norm.weight", "offset": 101675141120, "shape": [ 5120 ], "size": 540672000, "type": 0 } ], "version": 3 }