deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 16, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 790528000, "shape": [ 5120 ], "size": 360448000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 790548480, "shape": [ 12288, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 833802240, "shape": [ 5120, 12288 ], "size": 43253760, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 877056000, "shape": [ 5120, 12288 ], "size": 43253760, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 920309760, "shape": [ 5120 ], "size": 43253760, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 920330240, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 920332288, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.0.attn_kv_b.weight", "offset": 922359808, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 933894144, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 991565824, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 991571968, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.0.attn_q_b.weight", "offset": 996978688, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1022930944, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 1026207744, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 1037021184, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 1047834624, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 1058648064, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 1058650112, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_kv_b.weight", "offset": 1060677632, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 1072211968, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 1129883648, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1129889792, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.1.attn_q_b.weight", "offset": 1135296512, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "output_norm.weight", "offset": 1161248768, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1161269248, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1161289728, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 2026364928, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 2891440128, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 3756515328, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3756535808, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 3759812608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 3770626048, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 3781439488, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 3792252928, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 3792254976, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_kv_b.weight", "offset": 3794282496, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 3805816832, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 3863488512, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 3863494656, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.2.attn_q_b.weight", "offset": 3868901376, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 3894853632, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 3894874112, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 4759949312, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 5625024512, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 6490099712, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 6490120192, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 6493396992, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 6504210432, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 6515023872, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 6525837312, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 6525839360, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.3.attn_kv_b.weight", "offset": 6527866880, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 6539401216, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 6597072896, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 6597079040, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.3.attn_q_b.weight", "offset": 6602485760, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 6628438016, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 6628458496, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 7493533696, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 8358608896, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 9223684096, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 9223704576, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 9226981376, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 9237794816, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 9248608256, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 9259421696, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 9259423744, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_kv_b.weight", "offset": 9261451264, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 9272985600, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 9330657280, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 9330663424, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.4.attn_q_b.weight", "offset": 9336070144, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 9362022400, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 9362042880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 10227118080, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 11092193280, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 11957268480, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 11957288960, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 11957309440, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 12822384640, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 13687459840, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 14552535040, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 14555811840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 14566625280, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 14577438720, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 14588252160, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 14588272640, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 14588274688, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.5.attn_kv_b.weight", "offset": 14590302208, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 14601836544, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 14659508224, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 14659514368, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.5.attn_q_b.weight", "offset": 14664921088, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 14690873344, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 14694150144, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 14704963584, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 14715777024, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 14726590464, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 14726592512, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.6.attn_kv_b.weight", "offset": 14728620032, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 14740154368, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 14797826048, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 14797832192, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.6.attn_q_b.weight", "offset": 14803238912, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 14829191168, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 14829211648, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 15694286848, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 16559362048, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 17424437248, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 17424457728, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 17427734528, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 17438547968, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 17449361408, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 17460174848, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 17460176896, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.7.attn_kv_b.weight", "offset": 17462204416, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 17473738752, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 17531410432, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 17531416576, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.7.attn_q_b.weight", "offset": 17536823296, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 17562775552, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 17562796032, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 18427871232, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 19292946432, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 20158021632, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 20158042112, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 20161318912, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 20172132352, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 20182945792, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 20193759232, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 20193761280, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.8.attn_kv_b.weight", "offset": 20195788800, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 20207323136, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 20264994816, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 20265000960, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.8.attn_q_b.weight", "offset": 20270407680, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 20296359936, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 20296380416, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 21161455616, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 22026530816, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 22891606016, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 22891626496, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 22894903296, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 22905716736, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 22916530176, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 22927343616, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 22927345664, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.9.attn_kv_b.weight", "offset": 22929373184, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 22940907520, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 22998579200, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 22998585344, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.9.attn_q_b.weight", "offset": 23003992064, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 23029944320, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 23033221120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 23044034560, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 23054848000, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 23065661440, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 23065663488, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.10.attn_kv_b.weight", "offset": 23067691008, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 23079225344, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 23136897024, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 23136903168, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.10.attn_q_b.weight", "offset": 23142309888, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 23168262144, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 23168282624, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 24033357824, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 24898433024, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 25763508224, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 25763528704, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 25763549184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 26628624384, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 27493699584, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 28358774784, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 28358795264, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 28362072064, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 28372885504, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 28383698944, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 28394512384, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 28394514432, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.11.attn_kv_b.weight", "offset": 28396541952, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 28408076288, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 28465747968, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 28465754112, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.11.attn_q_b.weight", "offset": 28471160832, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 28497113088, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 28497133568, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 29362208768, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 30227283968, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 31092359168, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 31092379648, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 31095656448, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 31106469888, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 31117283328, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 31128096768, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 31128098816, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.12.attn_kv_b.weight", "offset": 31130126336, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 31141660672, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 31199332352, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 31199338496, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.12.attn_q_b.weight", "offset": 31204745216, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 31230697472, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 31230717952, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 32095793152, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 32960868352, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 33825943552, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 33825964032, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 33829240832, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 33840054272, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 33850867712, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 33861681152, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 33861683200, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.13.attn_kv_b.weight", "offset": 33863710720, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 33875245056, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 33932916736, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 33932922880, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.13.attn_q_b.weight", "offset": 33938329600, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 33964281856, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 33964302336, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 34829377536, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 35694452736, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 36559527936, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 36559548416, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 36562825216, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 36573638656, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 36584452096, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 36595265536, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 36595267584, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.14.attn_kv_b.weight", "offset": 36597295104, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 36608829440, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 36666501120, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 36666507264, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.14.attn_q_b.weight", "offset": 36671913984, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 36697866240, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 36697886720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 37562961920, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 38428037120, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 39293112320, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 39293132800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 39296409600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 39307223040, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 39318036480, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 39328849920, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 39328851968, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.15.attn_kv_b.weight", "offset": 39330879488, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 39342413824, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 39400085504, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 39400091648, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.15.attn_q_b.weight", "offset": 39405498368, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 39431450624, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 39431471104, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 40296546304, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 41161621504, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 42026696704, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 42026717184, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 42029993984, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 42040807424, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 42051620864, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 42062434304, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 42062436352, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.16.attn_kv_b.weight", "offset": 42064463872, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 42075998208, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 42133669888, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 42133676032, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.16.attn_q_b.weight", "offset": 42139082752, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 42165035008, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 42165055488, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 43030130688, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 43895205888, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 44760281088, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 44760301568, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 44763578368, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 44774391808, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 44785205248, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 44796018688, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 44796020736, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.17.attn_kv_b.weight", "offset": 44798048256, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 44809582592, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 44867254272, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 44867260416, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.17.attn_q_b.weight", "offset": 44872667136, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 44898619392, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 44898639872, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 45763715072, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 46628790272, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 47493865472, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 47493885952, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 47493906432, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 48358981632, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 49224056832, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 50089132032, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 50092408832, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 50103222272, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 50114035712, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 50124849152, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 50124869632, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 50124871680, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.18.attn_kv_b.weight", "offset": 50126899200, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 50138433536, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 50196105216, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 50196111360, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.18.attn_q_b.weight", "offset": 50201518080, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 50227470336, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 50230747136, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 50241560576, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 50252374016, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 50263187456, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 50263189504, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.19.attn_kv_b.weight", "offset": 50265217024, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 50276751360, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 50334423040, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 50334429184, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.19.attn_q_b.weight", "offset": 50339835904, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 50365788160, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 50365808640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 51230883840, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 52095959040, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 52961034240, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 52961054720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 52964331520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 52975144960, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 52985958400, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 52996771840, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 52996773888, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.20.attn_kv_b.weight", "offset": 52998801408, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 53010335744, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 53068007424, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 53068013568, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.20.attn_q_b.weight", "offset": 53073420288, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 53099372544, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 53099393024, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 53964468224, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 54829543424, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 55694618624, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 55694639104, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 55697915904, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 55708729344, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 55719542784, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 55730356224, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "\r\u0000\u0000\u0000\u0000\ufffd\ufffd\ufffd\f\u0000\u0000\u0000\u0017\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.21.", "offset": 55730358272, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.21.attn_kv_b.weight", "offset": 55732385792, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 55743920128, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 55801591808, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 55801597952, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.21.attn_q_b.weight", "offset": 55807004672, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 55832956928, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 55832977408, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 56698052608, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 57563127808, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 58428203008, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 58428223488, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 58431500288, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 58442313728, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 58453127168, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 58463940608, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 58463942656, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.22.attn_kv_b.weight", "offset": 58465970176, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 58477504512, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 58535176192, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 58535182336, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.22.attn_q_b.weight", "offset": 58540589056, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 58566541312, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 58566561792, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 59431636992, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 60296712192, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 61161787392, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 61161807872, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 61165084672, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 61175898112, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 61186711552, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 61197524992, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 61197527040, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.23.attn_kv_b.weight", "offset": 61199554560, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 61211088896, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 61268760576, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 61268766720, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.23.attn_q_b.weight", "offset": 61274173440, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 61300125696, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 61300146176, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 62165221376, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 63030296576, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 63895371776, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 63895392256, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 63898669056, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 63909482496, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 63920295936, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 63931109376, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 63931111424, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.24.attn_kv_b.weight", "offset": 63933138944, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 63944673280, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 64002344960, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 64002351104, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.24.attn_q_b.weight", "offset": 64007757824, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 64033710080, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 64033730560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 64898805760, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 65763880960, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 66628956160, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 66628976640, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 66632253440, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 66643066880, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 66653880320, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 66664693760, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 66664695808, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.25.attn_kv_b.weight", "offset": 66666723328, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 66678257664, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 66735929344, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 66735935488, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.25.attn_q_b.weight", "offset": 66741342208, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 66767294464, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 66767314944, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 67632390144, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 68497465344, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 69362540544, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 69362561024, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 69365837824, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 69376651264, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 69387464704, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 69398278144, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 69398280192, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.26.attn_kv_b.weight", "offset": 69400307712, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 69411842048, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 69469513728, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 69469519872, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.26.attn_q_b.weight", "offset": 69474926592, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 69500878848, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 69500899328, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 70365974528, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 71231049728, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 72096124928, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 72096145408, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 72099422208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 72110235648, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 72121049088, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 72131862528, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 72131864576, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.27.attn_kv_b.weight", "offset": 72133892096, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 72145426432, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 72203098112, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 72203104256, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.27.attn_q_b.weight", "offset": 72208510976, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 72234463232, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 72234483712, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 73099558912, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 73964634112, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 74829709312, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 74829729792, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 74833006592, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 74843820032, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 74854633472, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 74865446912, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 74865448960, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.28.attn_kv_b.weight", "offset": 74867476480, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 74879010816, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 74936682496, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 74936688640, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.28.attn_q_b.weight", "offset": 74942095360, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 74968047616, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 74968068096, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 75833143296, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 76698218496, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 77563293696, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 77563314176, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 77566590976, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 77577404416, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.29.ffn_up_shexp.weight", "offset": 77588217856, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 77599031296, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 77599033344, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.29.attn_kv_b.weight", "offset": 77601060864, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 77612595200, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 77670266880, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 77670273024, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.29.attn_q_b.weight", "offset": 77675679744, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 77701632000, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 77701652480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 78566727680, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 79431802880, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 80296878080, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 80296898560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 80300175360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 80310988800, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 80321802240, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 80332615680, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 80332617728, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.30.attn_kv_b.weight", "offset": 80334645248, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 80346179584, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 80403851264, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 80403857408, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.30.attn_q_b.weight", "offset": 80409264128, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 80435216384, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 80435236864, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 81300312064, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 82165387264, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 83030462464, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 83030482944, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 83030503424, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 83895578624, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 84760653824, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 85625729024, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 85629005824, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 85639819264, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 85650632704, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 85661446144, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 85661466624, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 85661468672, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.31.attn_kv_b.weight", "offset": 85663496192, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 85675030528, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 85732702208, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 85732708352, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.31.attn_q_b.weight", "offset": 85738115072, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 85764067328, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 85767344128, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 85778157568, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 85788971008, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 85799784448, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 85799786496, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.32.attn_kv_b.weight", "offset": 85801814016, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 85813348352, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 85871020032, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 85871026176, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.32.attn_q_b.weight", "offset": 85876432896, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 85902385152, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 85902405632, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 86767480832, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 87632556032, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 88497631232, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 88497651712, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 88500928512, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 88511741952, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 88522555392, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 88533368832, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 88533370880, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.33.attn_kv_b.weight", "offset": 88535398400, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 88546932736, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 88604604416, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 88604610560, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.33.attn_q_b.weight", "offset": 88610017280, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 88635969536, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 88635990016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 89501065216, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 90366140416, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 91231215616, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 91231236096, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 91234512896, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 91245326336, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 91256139776, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 91266953216, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 91266955264, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.34.attn_kv_b.weight", "offset": 91268982784, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 91280517120, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 91338188800, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 91338194944, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.34.attn_q_b.weight", "offset": 91343601664, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 91369553920, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 91369574400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 92234649600, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 93099724800, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 93964800000, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 93964820480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 93968097280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 93978910720, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 93989724160, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 94000537600, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 94000539648, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.35.attn_kv_b.weight", "offset": 94002567168, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 94014101504, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 94071773184, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 94071779328, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.35.attn_q_b.weight", "offset": 94077186048, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 94103138304, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 94103158784, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 94968233984, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 95833309184, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 96698384384, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 96698404864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 96701681664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 96712495104, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 96723308544, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 96734121984, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 96734124032, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.36.attn_kv_b.weight", "offset": 96736151552, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 96747685888, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 96805357568, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 96805363712, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.36.attn_q_b.weight", "offset": 96810770432, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 96836722688, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 96836743168, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 97701818368, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 98566893568, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 99431968768, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 99431989248, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 99435266048, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 99446079488, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 99456892928, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 99467706368, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 99467708416, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.37.attn_kv_b.weight", "offset": 99469735936, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 99481270272, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 99538941952, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 99538948096, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.37.attn_q_b.weight", "offset": 99544354816, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 99570307072, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 99570327552, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 100435402752, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 101300477952, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 102165553152, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 102165573632, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 102168850432, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 102179663872, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 102190477312, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 102201290752, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 102201292800, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.38.attn_kv_b.weight", "offset": 102203320320, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 102214854656, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 102272526336, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 102272532480, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.38.attn_q_b.weight", "offset": 102277939200, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 102303891456, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 102303911936, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 103168987136, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 104034062336, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 104899137536, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 104899158016, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 104902434816, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 104913248256, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 104924061696, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 104934875136, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 104934877184, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.39.attn_kv_b.weight", "offset": 104936904704, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 104948439040, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 105006110720, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 105006116864, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.39.attn_q_b.weight", "offset": 105011523584, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 105037475840, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 105037496320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 105902571520, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 106767646720, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 107632721920, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 107632742400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 107636019200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 107646832640, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 107657646080, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 107668459520, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 107668461568, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.40.attn_kv_b.weight", "offset": 107670489088, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 107682023424, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 107739695104, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 107739701248, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.40.attn_q_b.weight", "offset": 107745107968, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 107771060224, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 107771080704, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 108636155904, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 109501231104, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.40.ffn_norm.weight", "offset": 110366306304, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 110366326784, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 110369603584, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 110380417024, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 110391230464, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 110402043904, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 110402045952, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.41.attn_kv_b.weight", "offset": 110404073472, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 110415607808, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 110473279488, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 110473285632, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.41.attn_q_b.weight", "offset": 110478692352, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 110504644608, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 110504665088, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 111369740288, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 112234815488, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.41.ffn_norm.weight", "offset": 113099890688, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 113099911168, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 113103187968, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 113114001408, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 113124814848, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 113135628288, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 113135630336, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.42.attn_kv_b.weight", "offset": 113137657856, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.42.attn_output.weight", "offset": 113149192192, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 113206863872, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 113206870016, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.42.attn_q_b.weight", "offset": 113212276736, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.42.attn_norm.weight", "offset": 113238228992, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 113238249472, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 114103324672, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 114968399872, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.42.ffn_norm.weight", "offset": 115833475072, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 115833495552, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 115836772352, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 115847585792, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 115858399232, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 115869212672, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 115869214720, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.43.attn_kv_b.weight", "offset": 115871242240, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.43.attn_output.weight", "offset": 115882776576, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 115940448256, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 115940454400, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.43.attn_q_b.weight", "offset": 115945861120, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 115971813376, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 115971833856, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 116836909056, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 117701984256, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.43.ffn_norm.weight", "offset": 118567059456, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 118567079936, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 118567100416, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 119432175616, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 120297250816, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 121162326016, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 121165602816, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 121176416256, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 121187229696, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.44.ffn_norm.weight", "offset": 121198043136, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 121198063616, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 121198065664, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.44.attn_kv_b.weight", "offset": 121200093184, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.44.attn_output.weight", "offset": 121211627520, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 121269299200, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 121269305344, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.44.attn_q_b.weight", "offset": 121274712064, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 121300664320, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 121303941120, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 121314754560, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 121325568000, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 121336381440, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 121336383488, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.45.attn_kv_b.weight", "offset": 121338411008, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.45.attn_output.weight", "offset": 121349945344, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 121407617024, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 121407623168, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.45.attn_q_b.weight", "offset": 121413029888, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 121438982144, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 121439002624, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 122304077824, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 123169153024, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.45.ffn_norm.weight", "offset": 124034228224, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 124034248704, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 124037525504, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 124048338944, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 124059152384, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 124069965824, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 124069967872, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.46.attn_kv_b.weight", "offset": 124071995392, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.46.attn_output.weight", "offset": 124083529728, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 124141201408, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 124141207552, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.46.attn_q_b.weight", "offset": 124146614272, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.46.attn_norm.weight", "offset": 124172566528, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 124172587008, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 125037662208, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 125902737408, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.46.ffn_norm.weight", "offset": 126767812608, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 126767833088, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 126771109888, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 126781923328, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 126792736768, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 126803550208, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 126803552256, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.47.attn_kv_b.weight", "offset": 126805579776, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.47.attn_output.weight", "offset": 126817114112, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 126874785792, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 126874791936, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.47.attn_q_b.weight", "offset": 126880198656, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 126906150912, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 126906171392, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 127771246592, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 128636321792, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.47.ffn_norm.weight", "offset": 129501396992, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 129501417472, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 129504694272, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 129515507712, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 129526321152, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 129537134592, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 129537136640, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.48.attn_kv_b.weight", "offset": 129539164160, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.48.attn_output.weight", "offset": 129550698496, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 129608370176, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 129608376320, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.48.attn_q_b.weight", "offset": 129613783040, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 129639735296, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 129639755776, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 130504830976, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 131369906176, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.48.ffn_norm.weight", "offset": 132234981376, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 132235001856, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 132238278656, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 132249092096, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 132259905536, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 132270718976, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 132270721024, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.49.attn_kv_b.weight", "offset": 132272748544, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.49.attn_output.weight", "offset": 132284282880, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 132341954560, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 132341960704, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.49.attn_q_b.weight", "offset": 132347367424, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 132373319680, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 132373340160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 133238415360, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 134103490560, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.49.ffn_norm.weight", "offset": 134968565760, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 134968586240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 134971863040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 134982676480, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 134993489920, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 135004303360, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 135004305408, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.50.attn_kv_b.weight", "offset": 135006332928, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.50.attn_output.weight", "offset": 135017867264, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 135075538944, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 135075545088, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.50.attn_q_b.weight", "offset": 135080951808, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 135106904064, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 135106924544, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 135971999744, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 136837074944, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.50.ffn_norm.weight", "offset": 137702150144, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 137702170624, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 137705447424, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 137716260864, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 137727074304, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 137737887744, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 137737889792, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.51.attn_kv_b.weight", "offset": 137739917312, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.51.attn_output.weight", "offset": 137751451648, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 137809123328, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 137809129472, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.51.attn_q_b.weight", "offset": 137814536192, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.51.attn_norm.weight", "offset": 137840488448, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 137840508928, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 138705584128, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 139570659328, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.51.ffn_norm.weight", "offset": 140435734528, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 140435755008, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 140439031808, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 140449845248, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 140460658688, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 140471472128, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 140471474176, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.52.attn_kv_b.weight", "offset": 140473501696, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.52.attn_output.weight", "offset": 140485036032, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 140542707712, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 140542713856, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.52.attn_q_b.weight", "offset": 140548120576, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 140574072832, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 140574093312, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 141439168512, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 142304243712, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.52.ffn_norm.weight", "offset": 143169318912, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 143169339392, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 143172616192, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 143183429632, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 143194243072, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 143205056512, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 143205058560, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.53.attn_kv_b.weight", "offset": 143207086080, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.53.attn_output.weight", "offset": 143218620416, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 143276292096, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 143276298240, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.53.attn_q_b.weight", "offset": 143281704960, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.53.attn_norm.weight", "offset": 143307657216, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 143307677696, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 144172752896, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 145037828096, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.53.ffn_norm.weight", "offset": 145902903296, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 145902923776, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 145906200576, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 145917014016, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 145927827456, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 145938640896, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 145938642944, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.54.attn_kv_b.weight", "offset": 145940670464, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.54.attn_output.weight", "offset": 145952204800, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 146009876480, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 146009882624, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.54.attn_q_b.weight", "offset": 146015289344, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 146041241600, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 146041262080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 146906337280, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 147771412480, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.54.ffn_norm.weight", "offset": 148636487680, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 148636508160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 148639784960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 148650598400, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 148661411840, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 148672225280, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 148672227328, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.55.attn_kv_b.weight", "offset": 148674254848, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.55.attn_output.weight", "offset": 148685789184, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 148743460864, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 148743467008, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.55.attn_q_b.weight", "offset": 148748873728, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.55.attn_norm.weight", "offset": 148774825984, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 148774846464, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 149639921664, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 150504996864, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.55.ffn_norm.weight", "offset": 151370072064, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 151370092544, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 151373369344, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 151384182784, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 151394996224, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 151405809664, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 151405811712, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.56.attn_kv_b.weight", "offset": 151407839232, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.56.attn_output.weight", "offset": 151419373568, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 151477045248, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 151477051392, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.56.attn_q_b.weight", "offset": 151482458112, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 151508410368, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 151508430848, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 152373506048, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 153238581248, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.56.ffn_norm.weight", "offset": 154103656448, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 154103676928, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 154103697408, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 154968772608, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 155833847808, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 156698923008, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 156702199808, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 156713013248, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 156723826688, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.57.ffn_norm.weight", "offset": 156734640128, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 156734660608, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 156734662656, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.57.attn_kv_b.weight", "offset": 156736690176, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.57.attn_output.weight", "offset": 156748224512, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 156805896192, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 156805902336, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.57.attn_q_b.weight", "offset": 156811309056, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 156837261312, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 156840538112, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 156851351552, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 156862164992, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 156872978432, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 156872980480, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.58.attn_kv_b.weight", "offset": 156875008000, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.58.attn_output.weight", "offset": 156886542336, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 156944214016, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 156944220160, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.58.attn_q_b.weight", "offset": 156949626880, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 156975579136, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 156975599616, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 157840674816, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 158705750016, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.58.ffn_norm.weight", "offset": 159570825216, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 159570845696, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 159574122496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 159584935936, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 159595749376, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 159606562816, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 159606564864, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.59.attn_kv_b.weight", "offset": 159608592384, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.59.attn_output.weight", "offset": 159620126720, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 159677798400, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 159677804544, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.59.attn_q_b.weight", "offset": 159683211264, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 159709163520, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 159709184000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 160574259200, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 161439334400, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.59.ffn_norm.weight", "offset": 162304409600, "shape": [ 5120 ], "size": 865075200, "type": 0 } ], "version": 3 }