deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 17, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 790528000, "shape": [ 5120 ], "size": 360448000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 790548480, "shape": [ 12288, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 842158080, "shape": [ 5120, 12288 ], "size": 51609600, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 885411840, "shape": [ 5120, 12288 ], "size": 43253760, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 928665600, "shape": [ 5120 ], "size": 43253760, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 928686080, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 928688128, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.0.attn_kv_b.weight", "offset": 930715648, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 942249984, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 999921664, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 999927808, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.0.attn_q_b.weight", "offset": 1005334528, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1031286784, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 1034563584, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 1047465984, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 1058279424, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 1069092864, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 1069094912, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_kv_b.weight", "offset": 1071122432, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 1082656768, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 1140328448, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1140334592, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.1.attn_q_b.weight", "offset": 1145741312, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "output_norm.weight", "offset": 1171693568, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1171714048, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1171734528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 2203926528, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 3069001728, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 3934076928, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 3934097408, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 3937374208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 3950276608, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 3961090048, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 3971903488, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 3971905536, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_kv_b.weight", "offset": 3973933056, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 3985467392, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 4043139072, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 4043145216, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.2.attn_q_b.weight", "offset": 4048551936, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 4074504192, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 4074524672, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 5106716672, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 5971791872, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 6836867072, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 6836887552, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 6840164352, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 6853066752, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 6863880192, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 6874693632, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 6874695680, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.3.attn_kv_b.weight", "offset": 6876723200, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 6888257536, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 6945929216, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 6945935360, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.3.attn_q_b.weight", "offset": 6951342080, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 6977294336, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 6977314816, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 8009506816, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 8874582016, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 9739657216, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 9739677696, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 9742954496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 9755856896, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 9766670336, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 9777483776, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 9777485824, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_kv_b.weight", "offset": 9779513344, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 9791047680, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 9848719360, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 9848725504, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.4.attn_q_b.weight", "offset": 9854132224, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 9880084480, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 9880104960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 10912296960, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 11777372160, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 12642447360, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 12642467840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 12642488320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 13674680320, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 14539755520, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 15404830720, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 15408107520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 15421009920, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 15431823360, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 15442636800, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 15442657280, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 15442659328, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.5.attn_kv_b.weight", "offset": 15444686848, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 15456221184, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 15513892864, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 15513899008, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.5.attn_q_b.weight", "offset": 15519305728, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 15545257984, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 15548534784, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 15561437184, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 15572250624, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 15583064064, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 15583066112, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.6.attn_kv_b.weight", "offset": 15585093632, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 15596627968, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 15654299648, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 15654305792, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.6.attn_q_b.weight", "offset": 15659712512, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 15685664768, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 15685685248, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 16717877248, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 17582952448, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 18448027648, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 18448048128, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 18451324928, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 18462138368, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 18472951808, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 18483765248, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 18483767296, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.7.attn_kv_b.weight", "offset": 18485794816, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 18497329152, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 18555000832, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 18555006976, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.7.attn_q_b.weight", "offset": 18560413696, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 18586365952, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 18586386432, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 19451461632, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 20316536832, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 21181612032, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 21181632512, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 21184909312, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 21195722752, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 21206536192, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 21217349632, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 21217351680, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.8.attn_kv_b.weight", "offset": 21219379200, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 21230913536, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 21288585216, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 21288591360, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.8.attn_q_b.weight", "offset": 21293998080, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 21319950336, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 21319970816, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 22185046016, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 23050121216, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 23915196416, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 23915216896, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 23918493696, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 23931396096, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 23942209536, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 23953022976, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 23953025024, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.9.attn_kv_b.weight", "offset": 23955052544, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 23966586880, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 24024258560, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 24024264704, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.9.attn_q_b.weight", "offset": 24029671424, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 24055623680, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 24058900480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 24069713920, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 24080527360, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 24091340800, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 24091342848, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.10.attn_kv_b.weight", "offset": 24093370368, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 24104904704, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 24162576384, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 24162582528, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.10.attn_q_b.weight", "offset": 24167989248, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 24193941504, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 24193961984, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 25226153984, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 26091229184, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 26956304384, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 26956324864, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 26956345344, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 27821420544, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 28686495744, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 29551570944, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 29551591424, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 29554868224, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 29565681664, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 29576495104, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 29587308544, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 29587310592, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.11.attn_kv_b.weight", "offset": 29589338112, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 29600872448, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 29658544128, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 29658550272, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.11.attn_q_b.weight", "offset": 29663956992, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 29689909248, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 29689929728, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 30555004928, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 31420080128, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 32285155328, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 32285175808, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 32288452608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 32301355008, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 32312168448, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 32322981888, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 32322983936, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.12.attn_kv_b.weight", "offset": 32325011456, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 32336545792, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 32394217472, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 32394223616, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.12.attn_q_b.weight", "offset": 32399630336, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 32425582592, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 32425603072, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 33457795072, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 34322870272, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 35187945472, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 35187965952, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 35191242752, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 35202056192, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 35212869632, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 35223683072, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 35223685120, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "kv_a_norm.weight\u0001\u0000\u0000\u0000\u0000\u0002\u0000", "offset": 35225712640, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 35237246976, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 35294918656, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 35294924800, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.13.attn_q_b.weight", "offset": 35300331520, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 35326283776, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 35326304256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 36191379456, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 37056454656, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 37921529856, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 37921550336, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 37924827136, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 37935640576, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 37946454016, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 37957267456, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 37957269504, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.14.attn_kv_b.weight", "offset": 37959297024, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 37970831360, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 38028503040, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 38028509184, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.14.attn_q_b.weight", "offset": 38033915904, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 38059868160, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 38059888640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 38924963840, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 39790039040, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 40655114240, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 40655134720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 40658411520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 40671313920, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 40682127360, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 40692940800, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 40692942848, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.15.attn_kv_b.weight", "offset": 40694970368, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 40706504704, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 40764176384, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 40764182528, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.15.attn_q_b.weight", "offset": 40769589248, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 40795541504, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 40795561984, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 41827753984, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 42692829184, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 43557904384, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 43557924864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 43561201664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 43572015104, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 43582828544, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 43593641984, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 43593644032, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.16.attn_kv_b.weight", "offset": 43595671552, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 43607205888, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 43664877568, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 43664883712, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.16.attn_q_b.weight", "offset": 43670290432, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 43696242688, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 43696263168, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 44561338368, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 45426413568, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 46291488768, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 46291509248, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 46294786048, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 46305599488, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 46316412928, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 46327226368, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 46327228416, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.17.attn_kv_b.weight", "offset": 46329255936, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 46340790272, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 46398461952, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 46398468096, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.17.attn_q_b.weight", "offset": 46403874816, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 46429827072, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 46429847552, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 47294922752, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 48159997952, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 49025073152, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 49025093632, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 49025114112, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 50057306112, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 50922381312, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 51787456512, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 51790733312, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 51803635712, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 51814449152, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 51825262592, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 51825283072, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 51825285120, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.18.attn_kv_b.weight", "offset": 51827312640, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 51838846976, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 51896518656, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 51896524800, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.18.attn_q_b.weight", "offset": 51901931520, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 51927883776, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 51931160576, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 51941974016, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 51952787456, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 51963600896, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 51963602944, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.19.attn_kv_b.weight", "offset": 51965630464, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 51977164800, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 52034836480, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 52034842624, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.19.attn_q_b.weight", "offset": 52040249344, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 52066201600, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 52066222080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 52931297280, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 53796372480, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 54661447680, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 54661468160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 54664744960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 54675558400, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 54686371840, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 54697185280, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 54697187328, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.20.attn_kv_b.weight", "offset": 54699214848, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 54710749184, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 54768420864, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 54768427008, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.20.attn_q_b.weight", "offset": 54773833728, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 54799785984, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 54799806464, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 55664881664, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 56529956864, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 57395032064, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 57395052544, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 57398329344, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 57411231744, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 57422045184, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 57432858624, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "\r\u0000\u0000\u0000\u0000\ufffdC_\r\u0000\u0000\u0000\u0017\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.21.", "offset": 57432860672, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.21.attn_kv_b.weight", "offset": 57434888192, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 57446422528, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 57504094208, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 57504100352, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.21.attn_q_b.weight", "offset": 57509507072, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 57535459328, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 57535479808, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 58567671808, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 59432747008, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 60297822208, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 60297842688, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 60301119488, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 60311932928, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 60322746368, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 60333559808, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 60333561856, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.22.attn_kv_b.weight", "offset": 60335589376, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 60347123712, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 60404795392, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 60404801536, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.22.attn_q_b.weight", "offset": 60410208256, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 60436160512, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 60436180992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 61301256192, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 62166331392, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 63031406592, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 63031427072, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 63034703872, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 63045517312, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 63056330752, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 63067144192, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 63067146240, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.23.attn_kv_b.weight", "offset": 63069173760, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 63080708096, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 63138379776, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 63138385920, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.23.attn_q_b.weight", "offset": 63143792640, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 63169744896, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 63169765376, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 64034840576, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 64899915776, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 65764990976, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 65765011456, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 65768288256, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 65781190656, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 65792004096, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 65802817536, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 65802819584, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.24.attn_kv_b.weight", "offset": 65804847104, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 65816381440, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 65874053120, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 65874059264, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.24.attn_q_b.weight", "offset": 65879465984, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 65905418240, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 65905438720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 66937630720, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 67802705920, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 68667781120, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 68667801600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 68671078400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 68681891840, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 68692705280, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 68703518720, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 68703520768, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.25.attn_kv_b.weight", "offset": 68705548288, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 68717082624, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 68774754304, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 68774760448, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.25.attn_q_b.weight", "offset": 68780167168, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 68806119424, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 68806139904, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 69671215104, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 70536290304, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 71401365504, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 71401385984, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 71404662784, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 71415476224, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 71426289664, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 71437103104, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 71437105152, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.26.attn_kv_b.weight", "offset": 71439132672, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 71450667008, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 71508338688, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 71508344832, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.26.attn_q_b.weight", "offset": 71513751552, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 71539703808, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 71539724288, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 72404799488, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 73269874688, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 74134949888, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 74134970368, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 74138247168, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 74151149568, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 74161963008, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 74172776448, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 74172778496, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.27.attn_kv_b.weight", "offset": 74174806016, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 74186340352, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 74244012032, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 74244018176, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.27.attn_q_b.weight", "offset": 74249424896, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 74275377152, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 74275397632, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 75307589632, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 76172664832, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 77037740032, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 77037760512, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 77041037312, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 77051850752, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 77062664192, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 77073477632, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 77073479680, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.28.attn_kv_b.weight", "offset": 77075507200, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 77087041536, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 77144713216, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 77144719360, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.28.attn_q_b.weight", "offset": 77150126080, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 77176078336, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 77176098816, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 78041174016, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 78906249216, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 79771324416, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 79771344896, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 79774621696, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 79785435136, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.29.ffn_up_shexp.weight", "offset": 79796248576, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 79807062016, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 79807064064, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.29.attn_kv_b.weight", "offset": 79809091584, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 79820625920, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 79878297600, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 79878303744, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.29.attn_q_b.weight", "offset": 79883710464, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 79909662720, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 79909683200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 80774758400, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 81639833600, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 82504908800, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 82504929280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 82508206080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 82521108480, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 82531921920, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 82542735360, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 82542737408, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.30.attn_kv_b.weight", "offset": 82544764928, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 82556299264, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 82613970944, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 82613977088, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.30.attn_q_b.weight", "offset": 82619383808, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 82645336064, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 82645356544, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 83677548544, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 84542623744, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 85407698944, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 85407719424, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 85407739904, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 86272815104, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 87137890304, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 88002965504, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 88006242304, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 88017055744, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 88027869184, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 88038682624, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 88038703104, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 88038705152, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.31.attn_kv_b.weight", "offset": 88040732672, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.31.attn_output.weight", "offset": 88052267008, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 88109938688, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 88109944832, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.31.attn_q_b.weight", "offset": 88115351552, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 88141303808, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 88144580608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 88155394048, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 88166207488, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 88177020928, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 88177022976, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.32.attn_kv_b.weight", "offset": 88179050496, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 88190584832, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 88248256512, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 88248262656, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.32.attn_q_b.weight", "offset": 88253669376, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 88279621632, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 88279642112, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 89144717312, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 90009792512, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 90874867712, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 90874888192, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 90878164992, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 90891067392, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 90901880832, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 90912694272, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 90912696320, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.33.attn_kv_b.weight", "offset": 90914723840, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 90926258176, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 90983929856, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 90983936000, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.33.attn_q_b.weight", "offset": 90989342720, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 91015294976, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 91015315456, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 92047507456, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 92912582656, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 93777657856, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 93777678336, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 93780955136, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 93791768576, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 93802582016, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 93813395456, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 93813397504, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.34.attn_kv_b.weight", "offset": 93815425024, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 93826959360, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 93884631040, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 93884637184, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.34.attn_q_b.weight", "offset": 93890043904, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 93915996160, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 93916016640, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 94781091840, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 95646167040, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 96511242240, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 96511262720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 96514539520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 96525352960, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 96536166400, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 96546979840, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 96546981888, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.35.attn_kv_b.weight", "offset": 96549009408, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 96560543744, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 96618215424, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 96618221568, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.35.attn_q_b.weight", "offset": 96623628288, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 96649580544, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 96649601024, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 97514676224, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 98379751424, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 99244826624, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 99244847104, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 99248123904, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 99261026304, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 99271839744, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 99282653184, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 99282655232, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.36.attn_kv_b.weight", "offset": 99284682752, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 99296217088, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 99353888768, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 99353894912, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.36.attn_q_b.weight", "offset": 99359301632, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.36.attn_norm.weight", "offset": 99385253888, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 99385274368, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 100417466368, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 101282541568, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 102147616768, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 102147637248, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 102150914048, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 102161727488, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 102172540928, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 102183354368, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 102183356416, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.37.attn_kv_b.weight", "offset": 102185383936, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 102196918272, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 102254589952, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 102254596096, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.37.attn_q_b.weight", "offset": 102260002816, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 102285955072, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 102285975552, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 103151050752, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 104016125952, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 104881201152, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 104881221632, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 104884498432, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 104895311872, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 104906125312, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 104916938752, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 104916940800, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.38.attn_kv_b.weight", "offset": 104918968320, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 104930502656, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 104988174336, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 104988180480, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.38.attn_q_b.weight", "offset": 104993587200, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 105019539456, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 105019559936, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 105884635136, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 106749710336, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 107614785536, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 107614806016, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 107618082816, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 107630985216, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 107641798656, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 107652612096, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 107652614144, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.39.attn_kv_b.weight", "offset": 107654641664, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 107666176000, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 107723847680, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 107723853824, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.39.attn_q_b.weight", "offset": 107729260544, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 107755212800, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 107755233280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 108787425280, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 109652500480, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 110517575680, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 110517596160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 110520872960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 110531686400, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 110542499840, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 110553313280, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 110553315328, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.40.attn_kv_b.weight", "offset": 110555342848, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 110566877184, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 110624548864, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 110624555008, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.40.attn_q_b.weight", "offset": 110629961728, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 110655913984, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 110655934464, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 111521009664, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 112386084864, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.40.ffn_norm.weight", "offset": 113251160064, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 113251180544, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 113254457344, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 113265270784, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 113276084224, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 113286897664, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 113286899712, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.41.attn_kv_b.weight", "offset": 113288927232, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 113300461568, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 113358133248, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 113358139392, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.41.attn_q_b.weight", "offset": 113363546112, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 113389498368, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 113389518848, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 114254594048, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 115119669248, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.41.ffn_norm.weight", "offset": 115984744448, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 115984764928, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 115988041728, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 116000944128, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 116011757568, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 116022571008, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 116022573056, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.42.attn_kv_b.weight", "offset": 116024600576, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.42.attn_output.weight", "offset": 116036134912, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 116093806592, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 116093812736, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.42.attn_q_b.weight", "offset": 116099219456, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.42.attn_norm.weight", "offset": 116125171712, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 116125192192, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 117157384192, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 118022459392, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.42.ffn_norm.weight", "offset": 118887534592, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 118887555072, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 118890831872, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 118901645312, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 118912458752, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 118923272192, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 118923274240, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.43.attn_kv_b.weight", "offset": 118925301760, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.43.attn_output.weight", "offset": 118936836096, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 118994507776, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 118994513920, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.43.attn_q_b.weight", "offset": 118999920640, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.43.attn_norm.weight", "offset": 119025872896, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 119025893376, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 119890968576, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 120756043776, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.43.ffn_norm.weight", "offset": 121621118976, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 121621139456, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 121621159936, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 122486235136, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 123351310336, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 124216385536, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 124219662336, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 124230475776, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 124241289216, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.44.ffn_norm.weight", "offset": 124252102656, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 124252123136, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 124252125184, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.44.attn_kv_b.weight", "offset": 124254152704, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.44.attn_output.weight", "offset": 124265687040, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 124323358720, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 124323364864, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.44.attn_q_b.weight", "offset": 124328771584, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 124354723840, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 124358000640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 124370903040, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 124381716480, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 124392529920, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 124392531968, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.45.attn_kv_b.weight", "offset": 124394559488, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.45.attn_output.weight", "offset": 124406093824, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 124463765504, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 124463771648, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.45.attn_q_b.weight", "offset": 124469178368, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.45.attn_norm.weight", "offset": 124495130624, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 124495151104, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 125527343104, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 126392418304, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.45.ffn_norm.weight", "offset": 127257493504, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 127257513984, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 127260790784, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 127271604224, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 127282417664, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 127293231104, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 127293233152, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.46.attn_kv_b.weight", "offset": 127295260672, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.46.attn_output.weight", "offset": 127306795008, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 127364466688, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 127364472832, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.46.attn_q_b.weight", "offset": 127369879552, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.46.attn_norm.weight", "offset": 127395831808, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 127395852288, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 128260927488, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 129126002688, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.46.ffn_norm.weight", "offset": 129991077888, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 129991098368, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 129994375168, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 130005188608, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 130016002048, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 130026815488, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 130026817536, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.47.attn_kv_b.weight", "offset": 130028845056, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.47.attn_output.weight", "offset": 130040379392, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 130098051072, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 130098057216, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.47.attn_q_b.weight", "offset": 130103463936, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.47.attn_norm.weight", "offset": 130129416192, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 130129436672, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 130994511872, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 131859587072, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.47.ffn_norm.weight", "offset": 132724662272, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 132724682752, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 132727959552, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 132740861952, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 132751675392, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 132762488832, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 132762490880, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.48.attn_kv_b.weight", "offset": 132764518400, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.48.attn_output.weight", "offset": 132776052736, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 132833724416, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 132833730560, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.48.attn_q_b.weight", "offset": 132839137280, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.48.attn_norm.weight", "offset": 132865089536, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 132865110016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 133897302016, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 134762377216, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.48.ffn_norm.weight", "offset": 135627452416, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 135627472896, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 135630749696, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 135641563136, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 135652376576, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 135663190016, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 135663192064, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.49.attn_kv_b.weight", "offset": 135665219584, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.49.attn_output.weight", "offset": 135676753920, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 135734425600, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 135734431744, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.49.attn_q_b.weight", "offset": 135739838464, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.49.attn_norm.weight", "offset": 135765790720, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 135765811200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 136630886400, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 137495961600, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.49.ffn_norm.weight", "offset": 138361036800, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 138361057280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 138364334080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 13 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 138375147520, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 138385960960, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 138396774400, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 138396776448, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.50.attn_kv_b.weight", "offset": 138398803968, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.50.attn_output.weight", "offset": 138410338304, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 138468009984, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 138468016128, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.50.attn_q_b.weight", "offset": 138473422848, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.50.attn_norm.weight", "offset": 138499375104, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 138499395584, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 13 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 139364470784, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 140229545984, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.50.ffn_norm.weight", "offset": 141094621184, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 141094641664, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 141097918464, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 141110820864, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 141121634304, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 141132447744, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 141132449792, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.51.attn_kv_b.weight", "offset": 141134477312, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.51.attn_output.weight", "offset": 141146011648, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 141203683328, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 141203689472, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.51.attn_q_b.weight", "offset": 141209096192, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.51.attn_norm.weight", "offset": 141235048448, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 141235068928, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 142267260928, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 143132336128, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.51.ffn_norm.weight", "offset": 143997411328, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 143997431808, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 144000708608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 144013611008, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 144024424448, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 144035237888, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 144035239936, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.52.attn_kv_b.weight", "offset": 144037267456, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.52.attn_output.weight", "offset": 144048801792, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 144106473472, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 144106479616, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.52.attn_q_b.weight", "offset": 144111886336, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.52.attn_norm.weight", "offset": 144137838592, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 144137859072, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 145170051072, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 146035126272, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.52.ffn_norm.weight", "offset": 146900201472, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 146900221952, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 146903498752, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 146916401152, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 146927214592, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 146938028032, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 146938030080, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.53.attn_kv_b.weight", "offset": 146940057600, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.53.attn_output.weight", "offset": 146951591936, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 147009263616, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 147009269760, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.53.attn_q_b.weight", "offset": 147014676480, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.53.attn_norm.weight", "offset": 147040628736, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 147040649216, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 148072841216, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 148937916416, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.53.ffn_norm.weight", "offset": 149802991616, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 149803012096, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 149806288896, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 149819191296, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 149830004736, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 149840818176, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 149840820224, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.54.attn_kv_b.weight", "offset": 149842847744, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.54.attn_output.weight", "offset": 149854382080, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 149912053760, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 149912059904, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.54.attn_q_b.weight", "offset": 149917466624, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.54.attn_norm.weight", "offset": 149943418880, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 149943439360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 150975631360, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 151840706560, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.54.ffn_norm.weight", "offset": 152705781760, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 152705802240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 152709079040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 152721981440, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 152732794880, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 152743608320, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 152743610368, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.55.attn_kv_b.weight", "offset": 152745637888, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.55.attn_output.weight", "offset": 152757172224, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 152814843904, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 152814850048, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.55.attn_q_b.weight", "offset": 152820256768, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.55.attn_norm.weight", "offset": 152846209024, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 152846229504, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 153878421504, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 154743496704, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.55.ffn_norm.weight", "offset": 155608571904, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 155608592384, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 155611869184, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 155624771584, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 155635585024, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 155646398464, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 155646400512, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.56.attn_kv_b.weight", "offset": 155648428032, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.56.attn_output.weight", "offset": 155659962368, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 155717634048, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 155717640192, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.56.attn_q_b.weight", "offset": 155723046912, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.56.attn_norm.weight", "offset": 155748999168, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 155749019648, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 156781211648, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 157646286848, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.56.ffn_norm.weight", "offset": 158511362048, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 158511382528, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 158511403008, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 159543595008, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 160408670208, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 161273745408, "shape": [ 5120, 160 ], "size": 865075200, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 161277022208, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 161289924608, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 161300738048, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.57.ffn_norm.weight", "offset": 161311551488, "shape": [ 5120 ], "size": 10813440, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 161311571968, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 161311574016, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.57.attn_kv_b.weight", "offset": 161313601536, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.57.attn_output.weight", "offset": 161325135872, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 161382807552, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 161382813696, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.57.attn_q_b.weight", "offset": 161388220416, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 161414172672, "shape": [ 5120, 160 ], "size": 25952256, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 161417449472, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 161430351872, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 161441165312, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 161451978752, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 161451980800, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.58.attn_kv_b.weight", "offset": 161454008320, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.58.attn_output.weight", "offset": 161465542656, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 161523214336, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 161523220480, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.58.attn_q_b.weight", "offset": 161528627200, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.58.attn_norm.weight", "offset": 161554579456, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 161554599936, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 162586791936, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 163451867136, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.58.ffn_norm.weight", "offset": 164316942336, "shape": [ 5120 ], "size": 865075200, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 164316962816, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 164320239616, "shape": [ 3072, 5120 ], "size": 3276800, "type": 14 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 164333142016, "shape": [ 5120, 3072 ], "size": 12902400, "type": 13 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 164343955456, "shape": [ 5120, 3072 ], "size": 10813440, "type": 13 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 164354768896, "shape": [ 512 ], "size": 10813440, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 164354770944, "shape": [ 5120, 576 ], "size": 2048, "type": 13 }, { "name": "blk.59.attn_kv_b.weight", "offset": 164356798464, "shape": [ 512, 32768 ], "size": 2027520, "type": 13 }, { "name": "blk.59.attn_output.weight", "offset": 164368332800, "shape": [ 16384, 5120 ], "size": 11534336, "type": 13 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 164426004480, "shape": [ 1536 ], "size": 57671680, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 164426010624, "shape": [ 5120, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.59.attn_q_b.weight", "offset": 164431417344, "shape": [ 1536, 24576 ], "size": 5406720, "type": 13 }, { "name": "blk.59.attn_norm.weight", "offset": 164457369600, "shape": [ 5120 ], "size": 25952256, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 164457390080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 14 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 165489582080, "shape": [ 5120, 1536, 160 ], "size": 1032192000, "type": 13 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 166354657280, "shape": [ 5120, 1536, 160 ], "size": 865075200, "type": 13 }, { "name": "blk.59.ffn_norm.weight", "offset": 167219732480, "shape": [ 5120 ], "size": 865075200, "type": 0 } ], "version": 3 }