deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 7, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 557056000, "shape": [ 5120, 102400 ], "size": 557056000, "type": 8 }, { "name": "blk.0.attn_norm.weight", "offset": 1114112000, "shape": [ 5120 ], "size": 557056000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 1114132480, "shape": [ 12288, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 1180979200, "shape": [ 5120, 12288 ], "size": 66846720, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 1247825920, "shape": [ 5120, 12288 ], "size": 66846720, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 1314672640, "shape": [ 5120 ], "size": 66846720, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 1314693120, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 1314695168, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.0.attn_kv_b.weight", "offset": 1317828608, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 1335654400, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 1424783360, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 1424789504, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.0.attn_q_b.weight", "offset": 1433145344, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1473253376, "shape": [ 5120, 160 ], "size": 40108032, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 1476530176, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 1493241856, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 1509953536, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 1526665216, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 1526667264, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.1.attn_kv_b.weight", "offset": 1529800704, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1547626496, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 1636755456, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1636761600, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.1.attn_q_b.weight", "offset": 1645117440, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "output_norm.weight", "offset": 1685225472, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1685245952, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1685266432, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 3022200832, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 4359135232, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 5696069632, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 5696090112, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 5699366912, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 5716078592, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 5732790272, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 5749501952, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 5749504000, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.2.attn_kv_b.weight", "offset": 5752637440, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 5770463232, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 5859592192, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 5859598336, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.2.attn_q_b.weight", "offset": 5867954176, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 5908062208, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 5908082688, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 7245017088, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 8581951488, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 9918885888, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 9918906368, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 9922183168, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 9938894848, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 9955606528, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 9972318208, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 9972320256, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.3.attn_kv_b.weight", "offset": 9975453696, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 9993279488, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 10082408448, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 10082414592, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.3.attn_q_b.weight", "offset": 10090770432, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 10130878464, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 10130898944, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 11467833344, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 12804767744, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 14141702144, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 14141722624, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 14144999424, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 14161711104, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 14178422784, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 14195134464, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 14195136512, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.4.attn_kv_b.weight", "offset": 14198269952, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 14216095744, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 14305224704, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 14305230848, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.4.attn_q_b.weight", "offset": 14313586688, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 14353694720, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 14353715200, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 15690649600, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 17027584000, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 18364518400, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 18364538880, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 18364559360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 19701493760, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 21038428160, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 22375362560, "shape": [ 5120, 160 ], "size": 1336934400, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 22378639360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 22395351040, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 22412062720, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 22428774400, "shape": [ 5120 ], "size": 16711680, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 22428794880, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 22428796928, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.5.attn_kv_b.weight", "offset": 22431930368, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 22449756160, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 22538885120, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 22538891264, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.5.attn_q_b.weight", "offset": 22547247104, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 22587355136, "shape": [ 5120, 160 ], "size": 40108032, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 22590631936, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 22607343616, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 22624055296, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 22640766976, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 22640769024, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.6.attn_kv_b.weight", "offset": 22643902464, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 22661728256, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 22750857216, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 22750863360, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.6.attn_q_b.weight", "offset": 22759219200, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 22799327232, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 22799347712, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 24136282112, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 25473216512, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 26810150912, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 26810171392, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 26813448192, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 26830159872, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 26846871552, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 26863583232, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 26863585280, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.7.attn_kv_b.weight", "offset": 26866718720, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 26884544512, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 26973673472, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 26973679616, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.7.attn_q_b.weight", "offset": 26982035456, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 27022143488, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 27022163968, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 28359098368, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 29696032768, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 31032967168, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 31032987648, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 31036264448, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 31052976128, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 31069687808, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 31086399488, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 31086401536, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.8.attn_kv_b.weight", "offset": 31089534976, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 31107360768, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 31196489728, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 31196495872, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.8.attn_q_b.weight", "offset": 31204851712, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 31244959744, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 31244980224, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 32581914624, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 33918849024, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 35255783424, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 35255803904, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 35259080704, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 35275792384, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 35292504064, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 35309215744, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 35309217792, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.9.attn_kv_b.weight", "offset": 35312351232, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 35330177024, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 35419305984, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 35419312128, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.9.attn_q_b.weight", "offset": 35427667968, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 35467776000, "shape": [ 5120, 160 ], "size": 40108032, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 35471052800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 35487764480, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 35504476160, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 35521187840, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 35521189888, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.10.attn_kv_b.weight", "offset": 35524323328, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 35542149120, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 35631278080, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 35631284224, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.10.attn_q_b.weight", "offset": 35639640064, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 35679748096, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 35679768576, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 37016702976, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 38353637376, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 39690571776, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 39690592256, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 39690612736, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 41027547136, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 42364481536, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 43701415936, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 43701436416, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 43704713216, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 43721424896, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 43738136576, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 43754848256, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 43754850304, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.11.attn_kv_b.weight", "offset": 43757983744, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 43775809536, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 43864938496, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 43864944640, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.11.attn_q_b.weight", "offset": 43873300480, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 43913408512, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 43913428992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 45250363392, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 46587297792, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 47924232192, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 47924252672, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 47927529472, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 47944241152, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 47960952832, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 47977664512, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 47977666560, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.12.attn_kv_b.weight", "offset": 47980800000, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 47998625792, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 48087754752, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 48087760896, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.12.attn_q_b.weight", "offset": 48096116736, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 48136224768, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 48136245248, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 49473179648, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 50810114048, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 52147048448, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 52147068928, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 52150345728, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 52167057408, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 52183769088, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 52200480768, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 52200482816, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.13.attn_kv_b.weight", "offset": 52203616256, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 52221442048, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 52310571008, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 52310577152, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.13.attn_q_b.weight", "offset": 52318932992, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 52359041024, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 52359061504, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 53695995904, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 55032930304, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 56369864704, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 56369885184, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 56373161984, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 56389873664, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 56406585344, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 56423297024, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 56423299072, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.14.attn_kv_b.weight", "offset": 56426432512, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 56444258304, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 56533387264, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 56533393408, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.14.attn_q_b.weight", "offset": 56541749248, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 56581857280, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 56581877760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 57918812160, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 59255746560, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 60592680960, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 60592701440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 60595978240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 60612689920, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 60629401600, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 60646113280, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 60646115328, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.15.attn_kv_b.weight", "offset": 60649248768, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 60667074560, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 60756203520, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 60756209664, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.15.attn_q_b.weight", "offset": 60764565504, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 60804673536, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 60804694016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 62141628416, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 63478562816, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 64815497216, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 64815517696, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 64818794496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 64835506176, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 64852217856, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 64868929536, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 64868931584, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.16.attn_kv_b.weight", "offset": 64872065024, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 64889890816, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 64979019776, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 64979025920, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.16.attn_q_b.weight", "offset": 64987381760, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 65027489792, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 65027510272, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 66364444672, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 67701379072, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 69038313472, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 69038333952, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 69041610752, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 69058322432, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 69075034112, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 69091745792, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 69091747840, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.17.attn_kv_b.weight", "offset": 69094881280, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 69112707072, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 69201836032, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 69201842176, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.17.attn_q_b.weight", "offset": 69210198016, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 69250306048, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 69250326528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 70587260928, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 71924195328, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 73261129728, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 73261150208, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 73261170688, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 74598105088, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 75935039488, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 77271973888, "shape": [ 5120, 160 ], "size": 1336934400, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 77275250688, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 77291962368, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 77308674048, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 77325385728, "shape": [ 5120 ], "size": 16711680, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 77325406208, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 77325408256, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.18.attn_kv_b.weight", "offset": 77328541696, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 77346367488, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 77435496448, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 77435502592, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.18.attn_q_b.weight", "offset": 77443858432, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 77483966464, "shape": [ 5120, 160 ], "size": 40108032, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 77487243264, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 77503954944, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 77520666624, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 77537378304, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 77537380352, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.19.attn_kv_b.weight", "offset": 77540513792, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 77558339584, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 77647468544, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 77647474688, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.19.attn_q_b.weight", "offset": 77655830528, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 77695938560, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 77695959040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 79032893440, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 80369827840, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 81706762240, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 81706782720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 81710059520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 81726771200, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 81743482880, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 81760194560, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 81760196608, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.20.attn_kv_b.weight", "offset": 81763330048, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 81781155840, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 81870284800, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 81870290944, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.20.attn_q_b.weight", "offset": 81878646784, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 81918754816, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 81918775296, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 83255709696, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 84592644096, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 85929578496, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 85929598976, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 85932875776, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 85949587456, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 85966299136, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 85983010816, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "\b\u0000\u0000\u0000\u0000\ufffd\ufffd\u0004\u0014\u0000\u0000\u0000\u0017\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.21.", "offset": 85983012864, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.21.attn_kv_b.weight", "offset": 85986146304, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 86003972096, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 86093101056, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 86093107200, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.21.attn_q_b.weight", "offset": 86101463040, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 86141571072, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 86141591552, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 87478525952, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 88815460352, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 90152394752, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 90152415232, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 90155692032, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 90172403712, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 90189115392, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 90205827072, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 90205829120, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.22.attn_kv_b.weight", "offset": 90208962560, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 90226788352, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 90315917312, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 90315923456, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.22.attn_q_b.weight", "offset": 90324279296, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 90364387328, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 90364407808, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 91701342208, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 93038276608, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 94375211008, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 94375231488, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 94378508288, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 94395219968, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 94411931648, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 94428643328, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 94428645376, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.23.attn_kv_b.weight", "offset": 94431778816, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 94449604608, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 94538733568, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 94538739712, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.23.attn_q_b.weight", "offset": 94547095552, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 94587203584, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 94587224064, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 95924158464, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 97261092864, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 98598027264, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 98598047744, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 98601324544, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 98618036224, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 98634747904, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 98651459584, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 98651461632, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.24.attn_kv_b.weight", "offset": 98654595072, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 98672420864, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 98761549824, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 98761555968, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.24.attn_q_b.weight", "offset": 98769911808, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 98810019840, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 98810040320, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 100146974720, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 101483909120, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 102820843520, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 102820864000, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 102824140800, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 102840852480, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 102857564160, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 102874275840, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 102874277888, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.25.attn_kv_b.weight", "offset": 102877411328, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 102895237120, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 102984366080, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 102984372224, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.25.attn_q_b.weight", "offset": 102992728064, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 103032836096, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 103032856576, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 104369790976, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 105706725376, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 107043659776, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 107043680256, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 107046957056, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 107063668736, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 107080380416, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 107097092096, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 107097094144, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.26.attn_kv_b.weight", "offset": 107100227584, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 107118053376, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 107207182336, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 107207188480, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.26.attn_q_b.weight", "offset": 107215544320, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 107255652352, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 107255672832, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 108592607232, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 109929541632, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 111266476032, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 111266496512, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 111269773312, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 111286484992, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 111303196672, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 111319908352, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 111319910400, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.27.attn_kv_b.weight", "offset": 111323043840, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 111340869632, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 111429998592, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 111430004736, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.27.attn_q_b.weight", "offset": 111438360576, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 111478468608, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 111478489088, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 112815423488, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 114152357888, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 115489292288, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 115489312768, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 115492589568, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 115509301248, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 115526012928, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 115542724608, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 115542726656, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.28.attn_kv_b.weight", "offset": 115545860096, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 115563685888, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 115652814848, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 115652820992, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.28.attn_q_b.weight", "offset": 115661176832, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 115701284864, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 115701305344, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 117038239744, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 118375174144, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.28.ffn_norm.weight", "offset": 119712108544, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 119712129024, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 119715405824, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 119732117504, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "hexp.weight\u0002\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000", "offset": 119748829184, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 119765540864, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 119765542912, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.29.attn_kv_b.weight", "offset": 119768676352, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.29.attn_output.weight", "offset": 119786502144, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 119875631104, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 119875637248, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.29.attn_q_b.weight", "offset": 119883993088, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.29.attn_norm.weight", "offset": 119924101120, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 119924121600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 121261056000, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 122597990400, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.29.ffn_norm.weight", "offset": 123934924800, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 123934945280, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 123938222080, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 123954933760, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 123971645440, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 123988357120, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 123988359168, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.30.attn_kv_b.weight", "offset": 123991492608, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.30.attn_output.weight", "offset": 124009318400, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 124098447360, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 124098453504, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.30.attn_q_b.weight", "offset": 124106809344, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 124146917376, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 124146937856, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 125483872256, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 126820806656, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.30.ffn_norm.weight", "offset": 128157741056, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 128157761536, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 128157782016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 129494716416, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 130831650816, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 132168585216, "shape": [ 5120, 160 ], "size": 1336934400, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 132171862016, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 132188573696, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 132205285376, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.31.ffn_norm.weight", "offset": 132221997056, "shape": [ 5120 ], "size": 16711680, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 132222017536, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 132222019584, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.31.attn_kv_b.weight", "offset": 132225153024, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.31.attn_output.weight", "offset": 132242978816, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 132332107776, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 132332113920, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.31.attn_q_b.weight", "offset": 132340469760, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 132380577792, "shape": [ 5120, 160 ], "size": 40108032, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 132383854592, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 132400566272, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 132417277952, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 132433989632, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 132433991680, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.32.attn_kv_b.weight", "offset": 132437125120, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.32.attn_output.weight", "offset": 132454950912, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 132544079872, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 132544086016, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.32.attn_q_b.weight", "offset": 132552441856, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.32.attn_norm.weight", "offset": 132592549888, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 132592570368, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 133929504768, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 135266439168, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.32.ffn_norm.weight", "offset": 136603373568, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 136603394048, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 136606670848, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 136623382528, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 136640094208, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 136656805888, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 136656807936, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.33.attn_kv_b.weight", "offset": 136659941376, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.33.attn_output.weight", "offset": 136677767168, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 136766896128, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 136766902272, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.33.attn_q_b.weight", "offset": 136775258112, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.33.attn_norm.weight", "offset": 136815366144, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 136815386624, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 138152321024, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 139489255424, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.33.ffn_norm.weight", "offset": 140826189824, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 140826210304, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 140829487104, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 140846198784, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 140862910464, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 140879622144, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 140879624192, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.34.attn_kv_b.weight", "offset": 140882757632, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.34.attn_output.weight", "offset": 140900583424, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 140989712384, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 140989718528, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.34.attn_q_b.weight", "offset": 140998074368, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.34.attn_norm.weight", "offset": 141038182400, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 141038202880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 142375137280, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 143712071680, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.34.ffn_norm.weight", "offset": 145049006080, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 145049026560, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 145052303360, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 145069015040, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 145085726720, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 145102438400, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 145102440448, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.35.attn_kv_b.weight", "offset": 145105573888, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 145123399680, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 145212528640, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 145212534784, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.35.attn_q_b.weight", "offset": 145220890624, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.35.attn_norm.weight", "offset": 145260998656, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 145261019136, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 146597953536, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 147934887936, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.35.ffn_norm.weight", "offset": 149271822336, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 149271842816, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 149275119616, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 149291831296, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 149308542976, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 149325254656, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 149325256704, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.36.attn_kv_b.weight", "offset": 149328390144, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.36.attn_output.weight", "offset": 149346215936, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 149435344896, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 149435351040, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.36.attn_q_b.weight", "offset": 149443706880, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.36.attn_norm.weight", "offset": 149483814912, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 149483835392, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 150820769792, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 152157704192, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 153494638592, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 153494659072, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 153497935872, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 153514647552, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 153531359232, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 153548070912, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 153548072960, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.37.attn_kv_b.weight", "offset": 153551206400, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.37.attn_output.weight", "offset": 153569032192, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 153658161152, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 153658167296, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.37.attn_q_b.weight", "offset": 153666523136, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.37.attn_norm.weight", "offset": 153706631168, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 153706651648, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 155043586048, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 156380520448, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.37.ffn_norm.weight", "offset": 157717454848, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 157717475328, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 157720752128, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 157737463808, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 157754175488, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 157770887168, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 157770889216, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.38.attn_kv_b.weight", "offset": 157774022656, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.38.attn_output.weight", "offset": 157791848448, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 157880977408, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 157880983552, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.38.attn_q_b.weight", "offset": 157889339392, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.38.attn_norm.weight", "offset": 157929447424, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 157929467904, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 159266402304, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 160603336704, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.38.ffn_norm.weight", "offset": 161940271104, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 161940291584, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 161943568384, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 161960280064, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 161976991744, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 161993703424, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 161993705472, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.39.attn_kv_b.weight", "offset": 161996838912, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.39.attn_output.weight", "offset": 162014664704, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 162103793664, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 162103799808, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.39.attn_q_b.weight", "offset": 162112155648, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.39.attn_norm.weight", "offset": 162152263680, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 162152284160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 163489218560, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 164826152960, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.39.ffn_norm.weight", "offset": 166163087360, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 166163107840, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 166166384640, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 166183096320, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 166199808000, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 166216519680, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 166216521728, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.40.attn_kv_b.weight", "offset": 166219655168, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.40.attn_output.weight", "offset": 166237480960, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 166326609920, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 166326616064, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.40.attn_q_b.weight", "offset": 166334971904, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.40.attn_norm.weight", "offset": 166375079936, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 166375100416, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 167712034816, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 169048969216, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.40.ffn_norm.weight", "offset": 170385903616, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 170385924096, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 170389200896, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 170405912576, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 170422624256, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 170439335936, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 170439337984, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.41.attn_kv_b.weight", "offset": 170442471424, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.41.attn_output.weight", "offset": 170460297216, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 170549426176, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 170549432320, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.41.attn_q_b.weight", "offset": 170557788160, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.41.attn_norm.weight", "offset": 170597896192, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 170597916672, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 171934851072, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 173271785472, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.41.ffn_norm.weight", "offset": 174608719872, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 174608740352, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 174612017152, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 174628728832, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 174645440512, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 174662152192, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 174662154240, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.42.attn_kv_b.weight", "offset": 174665287680, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.42.attn_output.weight", "offset": 174683113472, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 174772242432, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 174772248576, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.42.attn_q_b.weight", "offset": 174780604416, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.42.attn_norm.weight", "offset": 174820712448, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 174820732928, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 176157667328, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 177494601728, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.42.ffn_norm.weight", "offset": 178831536128, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 178831556608, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 178834833408, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 178851545088, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 178868256768, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 178884968448, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 178884970496, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.43.attn_kv_b.weight", "offset": 178888103936, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.43.attn_output.weight", "offset": 178905929728, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 178995058688, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 178995064832, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.43.attn_q_b.weight", "offset": 179003420672, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.43.attn_norm.weight", "offset": 179043528704, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 179043549184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 180380483584, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 181717417984, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.43.ffn_norm.weight", "offset": 183054352384, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 183054372864, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 183054393344, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 184391327744, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 185728262144, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 187065196544, "shape": [ 5120, 160 ], "size": 1336934400, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 187068473344, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 187085185024, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 187101896704, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.44.ffn_norm.weight", "offset": 187118608384, "shape": [ 5120 ], "size": 16711680, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 187118628864, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 187118630912, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.44.attn_kv_b.weight", "offset": 187121764352, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.44.attn_output.weight", "offset": 187139590144, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 187228719104, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 187228725248, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.44.attn_q_b.weight", "offset": 187237081088, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 187277189120, "shape": [ 5120, 160 ], "size": 40108032, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 187280465920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 187297177600, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 187313889280, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 187330600960, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 187330603008, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.45.attn_kv_b.weight", "offset": 187333736448, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.45.attn_output.weight", "offset": 187351562240, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 187440691200, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 187440697344, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.45.attn_q_b.weight", "offset": 187449053184, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.45.attn_norm.weight", "offset": 187489161216, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 187489181696, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 188826116096, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 190163050496, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.45.ffn_norm.weight", "offset": 191499984896, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 191500005376, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 191503282176, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 191519993856, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 191536705536, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 191553417216, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 191553419264, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.46.attn_kv_b.weight", "offset": 191556552704, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.46.attn_output.weight", "offset": 191574378496, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 191663507456, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 191663513600, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.46.attn_q_b.weight", "offset": 191671869440, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.46.attn_norm.weight", "offset": 191711977472, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 191711997952, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 193048932352, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 194385866752, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.46.ffn_norm.weight", "offset": 195722801152, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 195722821632, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 195726098432, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 195742810112, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 195759521792, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 195776233472, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 195776235520, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.47.attn_kv_b.weight", "offset": 195779368960, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.47.attn_output.weight", "offset": 195797194752, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 195886323712, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 195886329856, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.47.attn_q_b.weight", "offset": 195894685696, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.47.attn_norm.weight", "offset": 195934793728, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 195934814208, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 197271748608, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 198608683008, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.47.ffn_norm.weight", "offset": 199945617408, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 199945637888, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 199948914688, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 199965626368, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 199982338048, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 199999049728, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 199999051776, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.48.attn_kv_b.weight", "offset": 200002185216, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.48.attn_output.weight", "offset": 200020011008, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 200109139968, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 200109146112, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.48.attn_q_b.weight", "offset": 200117501952, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.48.attn_norm.weight", "offset": 200157609984, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 200157630464, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 201494564864, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 202831499264, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.48.ffn_norm.weight", "offset": 204168433664, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 204168454144, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 204171730944, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 204188442624, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 204205154304, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 204221865984, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 204221868032, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.49.attn_kv_b.weight", "offset": 204225001472, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.49.attn_output.weight", "offset": 204242827264, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 204331956224, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 204331962368, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.49.attn_q_b.weight", "offset": 204340318208, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.49.attn_norm.weight", "offset": 204380426240, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 204380446720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 205717381120, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 207054315520, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.49.ffn_norm.weight", "offset": 208391249920, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 208391270400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 208394547200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 208411258880, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 208427970560, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 208444682240, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 208444684288, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.50.attn_kv_b.weight", "offset": 208447817728, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.50.attn_output.weight", "offset": 208465643520, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 208554772480, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 208554778624, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.50.attn_q_b.weight", "offset": 208563134464, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.50.attn_norm.weight", "offset": 208603242496, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 208603262976, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 209940197376, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 211277131776, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.50.ffn_norm.weight", "offset": 212614066176, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 212614086656, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 212617363456, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 212634075136, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 212650786816, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 212667498496, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 212667500544, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.51.attn_kv_b.weight", "offset": 212670633984, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.51.attn_output.weight", "offset": 212688459776, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 212777588736, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 212777594880, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.51.attn_q_b.weight", "offset": 212785950720, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.51.attn_norm.weight", "offset": 212826058752, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 212826079232, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 214163013632, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 215499948032, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.51.ffn_norm.weight", "offset": 216836882432, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 216836902912, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 216840179712, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 216856891392, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 216873603072, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 216890314752, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 216890316800, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.52.attn_kv_b.weight", "offset": 216893450240, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.52.attn_output.weight", "offset": 216911276032, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 217000404992, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 217000411136, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.52.attn_q_b.weight", "offset": 217008766976, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.52.attn_norm.weight", "offset": 217048875008, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 217048895488, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 218385829888, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 219722764288, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.52.ffn_norm.weight", "offset": 221059698688, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 221059719168, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 221062995968, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 221079707648, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 221096419328, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 221113131008, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 221113133056, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.53.attn_kv_b.weight", "offset": 221116266496, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.53.attn_output.weight", "offset": 221134092288, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 221223221248, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 221223227392, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.53.attn_q_b.weight", "offset": 221231583232, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.53.attn_norm.weight", "offset": 221271691264, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 221271711744, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 222608646144, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 223945580544, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.53.ffn_norm.weight", "offset": 225282514944, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 225282535424, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 225285812224, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 225302523904, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 225319235584, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 225335947264, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 225335949312, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.54.attn_kv_b.weight", "offset": 225339082752, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.54.attn_output.weight", "offset": 225356908544, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 225446037504, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 225446043648, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.54.attn_q_b.weight", "offset": 225454399488, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.54.attn_norm.weight", "offset": 225494507520, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 225494528000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 226831462400, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 228168396800, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.54.ffn_norm.weight", "offset": 229505331200, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 229505351680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 229508628480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 229525340160, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 229542051840, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 229558763520, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 229558765568, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.55.attn_kv_b.weight", "offset": 229561899008, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.55.attn_output.weight", "offset": 229579724800, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 229668853760, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 229668859904, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.55.attn_q_b.weight", "offset": 229677215744, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.55.attn_norm.weight", "offset": 229717323776, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 229717344256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 231054278656, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 232391213056, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.55.ffn_norm.weight", "offset": 233728147456, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 233728167936, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 233731444736, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 233748156416, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 233764868096, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 233781579776, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 233781581824, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.56.attn_kv_b.weight", "offset": 233784715264, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.56.attn_output.weight", "offset": 233802541056, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 233891670016, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 233891676160, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.56.attn_q_b.weight", "offset": 233900032000, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.56.attn_norm.weight", "offset": 233940140032, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 233940160512, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 235277094912, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 236614029312, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.56.ffn_norm.weight", "offset": 237950963712, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 237950984192, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 237951004672, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 239287939072, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 240624873472, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 241961807872, "shape": [ 5120, 160 ], "size": 1336934400, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 241965084672, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 241981796352, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 241998508032, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.57.ffn_norm.weight", "offset": 242015219712, "shape": [ 5120 ], "size": 16711680, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 242015240192, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 242015242240, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.57.attn_kv_b.weight", "offset": 242018375680, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.57.attn_output.weight", "offset": 242036201472, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 242125330432, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 242125336576, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.57.attn_q_b.weight", "offset": 242133692416, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 242173800448, "shape": [ 5120, 160 ], "size": 40108032, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 242177077248, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 242193788928, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 242210500608, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 242227212288, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 242227214336, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.58.attn_kv_b.weight", "offset": 242230347776, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.58.attn_output.weight", "offset": 242248173568, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 242337302528, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 242337308672, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.58.attn_q_b.weight", "offset": 242345664512, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.58.attn_norm.weight", "offset": 242385772544, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 242385793024, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 243722727424, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 245059661824, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.58.ffn_norm.weight", "offset": 246396596224, "shape": [ 5120 ], "size": 1336934400, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 246396616704, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 246399893504, "shape": [ 3072, 5120 ], "size": 3276800, "type": 8 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 246416605184, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 246433316864, "shape": [ 5120, 3072 ], "size": 16711680, "type": 8 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 246450028544, "shape": [ 512 ], "size": 16711680, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 246450030592, "shape": [ 5120, 576 ], "size": 2048, "type": 8 }, { "name": "blk.59.attn_kv_b.weight", "offset": 246453164032, "shape": [ 512, 32768 ], "size": 3133440, "type": 8 }, { "name": "blk.59.attn_output.weight", "offset": 246470989824, "shape": [ 16384, 5120 ], "size": 17825792, "type": 8 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 246560118784, "shape": [ 1536 ], "size": 89128960, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 246560124928, "shape": [ 5120, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.59.attn_q_b.weight", "offset": 246568480768, "shape": [ 1536, 24576 ], "size": 8355840, "type": 8 }, { "name": "blk.59.attn_norm.weight", "offset": 246608588800, "shape": [ 5120 ], "size": 40108032, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 246608609280, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 8 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 247945543680, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 249282478080, "shape": [ 5120, 1536, 160 ], "size": 1336934400, "type": 8 }, { "name": "blk.59.ffn_norm.weight", "offset": 250619412480, "shape": [ 5120 ], "size": 1336934400, "type": 0 } ], "version": 3 }