deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 1, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 1048576000, "shape": [ 5120, 102400 ], "size": 1048576000, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 2097152000, "shape": [ 5120 ], "size": 1048576000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 2097172480, "shape": [ 12288, 5120 ], "size": 20480, "type": 1 }, { "name": "blk.0.ffn_gate.weight", "offset": 2223001600, "shape": [ 5120, 12288 ], "size": 125829120, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 2348830720, "shape": [ 5120, 12288 ], "size": 125829120, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 2474659840, "shape": [ 5120 ], "size": 125829120, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 2474680320, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 2474682368, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.0.attn_kv_b.weight", "offset": 2480580608, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.0.attn_output.weight", "offset": 2514135040, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 2681907200, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 2681913344, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.0.attn_q_b.weight", "offset": 2697641984, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 2773139456, "shape": [ 5120, 160 ], "size": 75497472, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 2776416256, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 2807873536, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 2839330816, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 2870788096, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 2870790144, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.1.attn_kv_b.weight", "offset": 2876688384, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.1.attn_output.weight", "offset": 2910242816, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.1.attn_q_a_norm.weight", "offset": 3078014976, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 3078021120, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.1.attn_q_b.weight", "offset": 3093749760, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "output_norm.weight", "offset": 3169247232, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 3169267712, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 3169288192, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 5685870592, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 8202452992, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 10719035392, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 10719055872, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 10722332672, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 10753789952, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 10785247232, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 10816704512, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 10816706560, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.2.attn_kv_b.weight", "offset": 10822604800, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.2.attn_output.weight", "offset": 10856159232, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 11023931392, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 11023937536, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.2.attn_q_b.weight", "offset": 11039666176, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 11115163648, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 11115184128, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 13631766528, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 16148348928, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 18664931328, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 18664951808, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 18668228608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 18699685888, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 18731143168, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 18762600448, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 18762602496, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.3.attn_kv_b.weight", "offset": 18768500736, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.3.attn_output.weight", "offset": 18802055168, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 18969827328, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 18969833472, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.3.attn_q_b.weight", "offset": 18985562112, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 19061059584, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 19061080064, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 21577662464, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 24094244864, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 26610827264, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 26610847744, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 26614124544, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 26645581824, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 26677039104, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 26708496384, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 26708498432, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.4.attn_kv_b.weight", "offset": 26714396672, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.4.attn_output.weight", "offset": 26747951104, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 26915723264, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 26915729408, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.4.attn_q_b.weight", "offset": 26931458048, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 27006955520, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 27006976000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 29523558400, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 32040140800, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 34556723200, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 34556743680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 34556764160, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 37073346560, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 39589928960, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 42106511360, "shape": [ 5120, 160 ], "size": 2516582400, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 42109788160, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 42141245440, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 42172702720, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 42204160000, "shape": [ 5120 ], "size": 31457280, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 42204180480, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 42204182528, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.5.attn_kv_b.weight", "offset": 42210080768, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.5.attn_output.weight", "offset": 42243635200, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 42411407360, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 42411413504, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.5.attn_q_b.weight", "offset": 42427142144, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 42502639616, "shape": [ 5120, 160 ], "size": 75497472, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 42505916416, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 42537373696, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 42568830976, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 42600288256, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 42600290304, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.6.attn_kv_b.weight", "offset": 42606188544, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.6.attn_output.weight", "offset": 42639742976, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 42807515136, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 42807521280, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.6.attn_q_b.weight", "offset": 42823249920, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 42898747392, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 42898767872, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 45415350272, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 47931932672, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 50448515072, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 50448535552, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 50451812352, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 50483269632, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 50514726912, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 50546184192, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 50546186240, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.7.attn_kv_b.weight", "offset": 50552084480, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.7.attn_output.weight", "offset": 50585638912, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 50753411072, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 50753417216, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.7.attn_q_b.weight", "offset": 50769145856, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 50844643328, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 50844663808, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 53361246208, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 55877828608, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 58394411008, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 58394431488, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 58397708288, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 58429165568, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 58460622848, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 58492080128, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 58492082176, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.8.attn_kv_b.weight", "offset": 58497980416, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.8.attn_output.weight", "offset": 58531534848, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 58699307008, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 58699313152, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.8.attn_q_b.weight", "offset": 58715041792, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 58790539264, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 58790559744, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 61307142144, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 63823724544, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 66340306944, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 66340327424, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 66343604224, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 66375061504, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 66406518784, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 66437976064, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 66437978112, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.9.attn_kv_b.weight", "offset": 66443876352, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.9.attn_output.weight", "offset": 66477430784, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 66645202944, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 66645209088, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.9.attn_q_b.weight", "offset": 66660937728, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 66736435200, "shape": [ 5120, 160 ], "size": 75497472, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 66739712000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 66771169280, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 66802626560, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 66834083840, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 66834085888, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.10.attn_kv_b.weight", "offset": 66839984128, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.10.attn_output.weight", "offset": 66873538560, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 67041310720, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 67041316864, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.10.attn_q_b.weight", "offset": 67057045504, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 67132542976, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 67132563456, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 69649145856, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 72165728256, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 74682310656, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 74682331136, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 74682351616, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 77198934016, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 79715516416, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 82232098816, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 82232119296, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 82235396096, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 82266853376, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 82298310656, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 82329767936, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 82329769984, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.11.attn_kv_b.weight", "offset": 82335668224, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.11.attn_output.weight", "offset": 82369222656, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 82536994816, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 82537000960, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.11.attn_q_b.weight", "offset": 82552729600, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 82628227072, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 82628247552, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 85144829952, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 87661412352, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 90177994752, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 90178015232, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 90181292032, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 90212749312, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 90244206592, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 90275663872, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 90275665920, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.12.attn_kv_b.weight", "offset": 90281564160, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.12.attn_output.weight", "offset": 90315118592, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 90482890752, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 90482896896, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.12.attn_q_b.weight", "offset": 90498625536, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 90574123008, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 90574143488, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 93090725888, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 95607308288, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 98123890688, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 98123911168, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 98127187968, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 98158645248, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 98190102528, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 98221559808, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 98221561856, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "kv_a_norm.weight\u0001\u0000\u0000\u0000\u0000\u0002\u0000", "offset": 98227460096, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.13.attn_output.weight", "offset": 98261014528, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 98428786688, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 98428792832, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.13.attn_q_b.weight", "offset": 98444521472, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 98520018944, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 98520039424, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 101036621824, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 103553204224, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 106069786624, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 106069807104, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 106073083904, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 106104541184, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 106135998464, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 106167455744, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 106167457792, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.14.attn_kv_b.weight", "offset": 106173356032, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.14.attn_output.weight", "offset": 106206910464, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 106374682624, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 106374688768, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.14.attn_q_b.weight", "offset": 106390417408, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 106465914880, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 106465935360, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 108982517760, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 111499100160, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 114015682560, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 114015703040, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 114018979840, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 114050437120, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 114081894400, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 114113351680, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 114113353728, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.15.attn_kv_b.weight", "offset": 114119251968, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.15.attn_output.weight", "offset": 114152806400, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 114320578560, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 114320584704, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.15.attn_q_b.weight", "offset": 114336313344, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 114411810816, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 114411831296, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 116928413696, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 119444996096, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 121961578496, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 121961598976, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 121964875776, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 121996333056, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 122027790336, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 122059247616, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 122059249664, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.16.attn_kv_b.weight", "offset": 122065147904, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.16.attn_output.weight", "offset": 122098702336, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 122266474496, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 122266480640, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.16.attn_q_b.weight", "offset": 122282209280, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 122357706752, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 122357727232, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 124874309632, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 127390892032, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 129907474432, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 129907494912, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 129910771712, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 129942228992, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 129973686272, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 130005143552, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 130005145600, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.17.attn_kv_b.weight", "offset": 130011043840, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.17.attn_output.weight", "offset": 130044598272, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 130212370432, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 130212376576, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.17.attn_q_b.weight", "offset": 130228105216, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 130303602688, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 130303623168, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 132820205568, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 135336787968, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 137853370368, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 137853390848, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 137853411328, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 140369993728, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 142886576128, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 145403158528, "shape": [ 5120, 160 ], "size": 2516582400, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 145406435328, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 145437892608, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 145469349888, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 145500807168, "shape": [ 5120 ], "size": 31457280, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 145500827648, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 145500829696, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.18.attn_kv_b.weight", "offset": 145506727936, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.18.attn_output.weight", "offset": 145540282368, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 145708054528, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 145708060672, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.18.attn_q_b.weight", "offset": 145723789312, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 145799286784, "shape": [ 5120, 160 ], "size": 75497472, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 145802563584, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 145834020864, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 145865478144, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 145896935424, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 145896937472, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.19.attn_kv_b.weight", "offset": 145902835712, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.19.attn_output.weight", "offset": 145936390144, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 146104162304, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 146104168448, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.19.attn_q_b.weight", "offset": 146119897088, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 146195394560, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 146195415040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 148711997440, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 151228579840, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 153745162240, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 153745182720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 153748459520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 153779916800, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 153811374080, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 153842831360, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 153842833408, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.20.attn_kv_b.weight", "offset": 153848731648, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.20.attn_output.weight", "offset": 153882286080, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 154050058240, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 154050064384, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.20.attn_q_b.weight", "offset": 154065793024, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 154141290496, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 154141310976, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 156657893376, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 159174475776, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 161691058176, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 161691078656, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 161694355456, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 161725812736, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 161757270016, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 161788727296, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "\u0017\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.21.attn_kv_b.we", "offset": 161788729344, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.21.attn_kv_b.weight", "offset": 161794627584, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 161828182016, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 161995954176, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 161995960320, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.21.attn_q_b.weight", "offset": 162011688960, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 162087186432, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 162087206912, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 164603789312, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 167120371712, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 169636954112, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 169636974592, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 169640251392, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 169671708672, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 169703165952, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 169734623232, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 169734625280, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.22.attn_kv_b.weight", "offset": 169740523520, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.22.attn_output.weight", "offset": 169774077952, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 169941850112, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 169941856256, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.22.attn_q_b.weight", "offset": 169957584896, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 170033082368, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 170033102848, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 172549685248, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 175066267648, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 177582850048, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 177582870528, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 177586147328, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 177617604608, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 177649061888, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 177680519168, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 177680521216, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.23.attn_kv_b.weight", "offset": 177686419456, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.23.attn_output.weight", "offset": 177719973888, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 177887746048, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 177887752192, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.23.attn_q_b.weight", "offset": 177903480832, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 177978978304, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 177978998784, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 180495581184, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 183012163584, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 185528745984, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 185528766464, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 185532043264, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 185563500544, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 185594957824, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 185626415104, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 185626417152, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.24.attn_kv_b.weight", "offset": 185632315392, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.24.attn_output.weight", "offset": 185665869824, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 185833641984, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 185833648128, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.24.attn_q_b.weight", "offset": 185849376768, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 185924874240, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 185924894720, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 188441477120, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 190958059520, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 193474641920, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 193474662400, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 193477939200, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 193509396480, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 193540853760, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 193572311040, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 193572313088, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.25.attn_kv_b.weight", "offset": 193578211328, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.25.attn_output.weight", "offset": 193611765760, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 193779537920, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 193779544064, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.25.attn_q_b.weight", "offset": 193795272704, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 193870770176, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 193870790656, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 196387373056, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 198903955456, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 201420537856, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 201420558336, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 201423835136, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 201455292416, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 201486749696, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 201518206976, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 201518209024, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.26.attn_kv_b.weight", "offset": 201524107264, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.26.attn_output.weight", "offset": 201557661696, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 201725433856, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 201725440000, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.26.attn_q_b.weight", "offset": 201741168640, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 201816666112, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 201816686592, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 204333268992, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 206849851392, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 209366433792, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 209366454272, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 209369731072, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 209401188352, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 209432645632, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 209464102912, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 209464104960, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.27.attn_kv_b.weight", "offset": 209470003200, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.27.attn_output.weight", "offset": 209503557632, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 209671329792, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 209671335936, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.27.attn_q_b.weight", "offset": 209687064576, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 209762562048, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 209762582528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 212279164928, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 214795747328, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 217312329728, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 217312350208, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 217315627008, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 217347084288, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 217378541568, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 217409998848, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 217410000896, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.28.attn_kv_b.weight", "offset": 217415899136, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.28.attn_output.weight", "offset": 217449453568, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 217617225728, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 217617231872, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.28.attn_q_b.weight", "offset": 217632960512, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 217708457984, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 217708478464, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 220225060864, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 222741643264, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 225258225664, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 225258246144, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 225261522944, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 225292980224, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "hexp.weight\u0002\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000", "offset": 225324437504, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 225355894784, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 225355896832, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.29.attn_kv_b.weight", "offset": 225361795072, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.29.attn_output.weight", "offset": 225395349504, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 225563121664, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 225563127808, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.29.attn_q_b.weight", "offset": 225578856448, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 225654353920, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 225654374400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 228170956800, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 230687539200, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 233204121600, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 233204142080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 233207418880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 233238876160, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 233270333440, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 233301790720, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 233301792768, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.30.attn_kv_b.weight", "offset": 233307691008, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.30.attn_output.weight", "offset": 233341245440, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 233509017600, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 233509023744, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.30.attn_q_b.weight", "offset": 233524752384, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 233600249856, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 233600270336, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 236116852736, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 238633435136, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 241150017536, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 241150038016, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 241150058496, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 243666640896, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 246183223296, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 248699805696, "shape": [ 5120, 160 ], "size": 2516582400, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 248703082496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 248734539776, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 248765997056, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 248797454336, "shape": [ 5120 ], "size": 31457280, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 248797474816, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 248797476864, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.31.attn_kv_b.weight", "offset": 248803375104, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.31.attn_output.weight", "offset": 248836929536, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 249004701696, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 249004707840, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.31.attn_q_b.weight", "offset": 249020436480, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 249095933952, "shape": [ 5120, 160 ], "size": 75497472, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 249099210752, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 249130668032, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 249162125312, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 249193582592, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 249193584640, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.32.attn_kv_b.weight", "offset": 249199482880, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.32.attn_output.weight", "offset": 249233037312, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 249400809472, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 249400815616, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.32.attn_q_b.weight", "offset": 249416544256, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.32.attn_norm.weight", "offset": 249492041728, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 249492062208, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 252008644608, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 254525227008, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.32.ffn_norm.weight", "offset": 257041809408, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 257041829888, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 257045106688, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 257076563968, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 257108021248, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 257139478528, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 257139480576, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.33.attn_kv_b.weight", "offset": 257145378816, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.33.attn_output.weight", "offset": 257178933248, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 257346705408, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 257346711552, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.33.attn_q_b.weight", "offset": 257362440192, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.33.attn_norm.weight", "offset": 257437937664, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 257437958144, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 259954540544, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 262471122944, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.33.ffn_norm.weight", "offset": 264987705344, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 264987725824, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 264991002624, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 265022459904, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 265053917184, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 265085374464, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 265085376512, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.34.attn_kv_b.weight", "offset": 265091274752, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.34.attn_output.weight", "offset": 265124829184, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 265292601344, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 265292607488, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.34.attn_q_b.weight", "offset": 265308336128, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.34.attn_norm.weight", "offset": 265383833600, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 265383854080, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 267900436480, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 270417018880, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.34.ffn_norm.weight", "offset": 272933601280, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 272933621760, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 272936898560, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 272968355840, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 272999813120, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 273031270400, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 273031272448, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.35.attn_kv_b.weight", "offset": 273037170688, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.35.attn_output.weight", "offset": 273070725120, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 273238497280, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 273238503424, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.35.attn_q_b.weight", "offset": 273254232064, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.35.attn_norm.weight", "offset": 273329729536, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 273329750016, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 275846332416, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 278362914816, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.35.ffn_norm.weight", "offset": 280879497216, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 280879517696, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 280882794496, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 280914251776, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 280945709056, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 280977166336, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 280977168384, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.36.attn_kv_b.weight", "offset": 280983066624, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.36.attn_output.weight", "offset": 281016621056, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 281184393216, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 281184399360, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.36.attn_q_b.weight", "offset": 281200128000, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.36.attn_norm.weight", "offset": 281275625472, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 281275645952, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 283792228352, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 286308810752, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.36.ffn_norm.weight", "offset": 288825393152, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 288825413632, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 288828690432, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 288860147712, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 288891604992, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 288923062272, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 288923064320, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.37.attn_kv_b.weight", "offset": 288928962560, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.37.attn_output.weight", "offset": 288962516992, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 289130289152, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 289130295296, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.37.attn_q_b.weight", "offset": 289146023936, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.37.attn_norm.weight", "offset": 289221521408, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 289221541888, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 291738124288, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 294254706688, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.37.ffn_norm.weight", "offset": 296771289088, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 296771309568, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 296774586368, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 296806043648, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 296837500928, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 296868958208, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 296868960256, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.38.attn_kv_b.weight", "offset": 296874858496, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.38.attn_output.weight", "offset": 296908412928, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 297076185088, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 297076191232, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.38.attn_q_b.weight", "offset": 297091919872, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.38.attn_norm.weight", "offset": 297167417344, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 297167437824, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 299684020224, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 302200602624, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.38.ffn_norm.weight", "offset": 304717185024, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 304717205504, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 304720482304, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 304751939584, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 304783396864, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 304814854144, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 304814856192, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.39.attn_kv_b.weight", "offset": 304820754432, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.39.attn_output.weight", "offset": 304854308864, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 305022081024, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 305022087168, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.39.attn_q_b.weight", "offset": 305037815808, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.39.attn_norm.weight", "offset": 305113313280, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 305113333760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 307629916160, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 310146498560, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.39.ffn_norm.weight", "offset": 312663080960, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 312663101440, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 312666378240, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 312697835520, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 312729292800, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 312760750080, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 312760752128, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.40.attn_kv_b.weight", "offset": 312766650368, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.40.attn_output.weight", "offset": 312800204800, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 312967976960, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 312967983104, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.40.attn_q_b.weight", "offset": 312983711744, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.40.attn_norm.weight", "offset": 313059209216, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 313059229696, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 315575812096, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 318092394496, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.40.ffn_norm.weight", "offset": 320608976896, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 320608997376, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 320612274176, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 320643731456, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 320675188736, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 320706646016, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 320706648064, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.41.attn_kv_b.weight", "offset": 320712546304, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.41.attn_output.weight", "offset": 320746100736, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 320913872896, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 320913879040, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.41.attn_q_b.weight", "offset": 320929607680, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.41.attn_norm.weight", "offset": 321005105152, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 321005125632, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 323521708032, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 326038290432, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.41.ffn_norm.weight", "offset": 328554872832, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 328554893312, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 328558170112, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 328589627392, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 328621084672, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 328652541952, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 328652544000, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.42.attn_kv_b.weight", "offset": 328658442240, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.42.attn_output.weight", "offset": 328691996672, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 328859768832, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 328859774976, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.42.attn_q_b.weight", "offset": 328875503616, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.42.attn_norm.weight", "offset": 328951001088, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 328951021568, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 331467603968, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 333984186368, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.42.ffn_norm.weight", "offset": 336500768768, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 336500789248, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 336504066048, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 336535523328, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 336566980608, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 336598437888, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 336598439936, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.43.attn_kv_b.weight", "offset": 336604338176, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.43.attn_output.weight", "offset": 336637892608, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 336805664768, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 336805670912, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.43.attn_q_b.weight", "offset": 336821399552, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.43.attn_norm.weight", "offset": 336896897024, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 336896917504, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 339413499904, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 341930082304, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.43.ffn_norm.weight", "offset": 344446664704, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 344446685184, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 344446705664, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 346963288064, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 349479870464, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 351996452864, "shape": [ 5120, 160 ], "size": 2516582400, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 351999729664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 352031186944, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 352062644224, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.44.ffn_norm.weight", "offset": 352094101504, "shape": [ 5120 ], "size": 31457280, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 352094121984, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 352094124032, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.44.attn_kv_b.weight", "offset": 352100022272, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.44.attn_output.weight", "offset": 352133576704, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 352301348864, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 352301355008, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.44.attn_q_b.weight", "offset": 352317083648, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 352392581120, "shape": [ 5120, 160 ], "size": 75497472, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 352395857920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 352427315200, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 352458772480, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 352490229760, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 352490231808, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.45.attn_kv_b.weight", "offset": 352496130048, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.45.attn_output.weight", "offset": 352529684480, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 352697456640, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 352697462784, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.45.attn_q_b.weight", "offset": 352713191424, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.45.attn_norm.weight", "offset": 352788688896, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 352788709376, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 355305291776, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 357821874176, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.45.ffn_norm.weight", "offset": 360338456576, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 360338477056, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 360341753856, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 360373211136, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 360404668416, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 360436125696, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 360436127744, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.46.attn_kv_b.weight", "offset": 360442025984, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.46.attn_output.weight", "offset": 360475580416, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 360643352576, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 360643358720, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.46.attn_q_b.weight", "offset": 360659087360, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.46.attn_norm.weight", "offset": 360734584832, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 360734605312, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 363251187712, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 365767770112, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.46.ffn_norm.weight", "offset": 368284352512, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 368284372992, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 368287649792, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 368319107072, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 368350564352, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 368382021632, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 368382023680, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.47.attn_kv_b.weight", "offset": 368387921920, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.47.attn_output.weight", "offset": 368421476352, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 368589248512, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 368589254656, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.47.attn_q_b.weight", "offset": 368604983296, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.47.attn_norm.weight", "offset": 368680480768, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 368680501248, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 371197083648, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 373713666048, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.47.ffn_norm.weight", "offset": 376230248448, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 376230268928, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 376233545728, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 376265003008, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 376296460288, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 376327917568, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 376327919616, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.48.attn_kv_b.weight", "offset": 376333817856, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.48.attn_output.weight", "offset": 376367372288, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 376535144448, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 376535150592, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.48.attn_q_b.weight", "offset": 376550879232, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.48.attn_norm.weight", "offset": 376626376704, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 376626397184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 379142979584, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 381659561984, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.48.ffn_norm.weight", "offset": 384176144384, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 384176164864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 384179441664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 384210898944, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 384242356224, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 384273813504, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 384273815552, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.49.attn_kv_b.weight", "offset": 384279713792, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.49.attn_output.weight", "offset": 384313268224, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 384481040384, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 384481046528, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.49.attn_q_b.weight", "offset": 384496775168, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.49.attn_norm.weight", "offset": 384572272640, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 384572293120, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 387088875520, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 389605457920, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.49.ffn_norm.weight", "offset": 392122040320, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 392122060800, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 392125337600, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 392156794880, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 392188252160, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 392219709440, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 392219711488, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.50.attn_kv_b.weight", "offset": 392225609728, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.50.attn_output.weight", "offset": 392259164160, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 392426936320, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 392426942464, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.50.attn_q_b.weight", "offset": 392442671104, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.50.attn_norm.weight", "offset": 392518168576, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 392518189056, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 395034771456, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 397551353856, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.50.ffn_norm.weight", "offset": 400067936256, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 400067956736, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 400071233536, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 400102690816, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 400134148096, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 400165605376, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 400165607424, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.51.attn_kv_b.weight", "offset": 400171505664, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.51.attn_output.weight", "offset": 400205060096, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 400372832256, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 400372838400, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.51.attn_q_b.weight", "offset": 400388567040, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.51.attn_norm.weight", "offset": 400464064512, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 400464084992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 402980667392, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 405497249792, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.51.ffn_norm.weight", "offset": 408013832192, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 408013852672, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 408017129472, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 408048586752, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 408080044032, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 408111501312, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 408111503360, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.52.attn_kv_b.weight", "offset": 408117401600, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.52.attn_output.weight", "offset": 408150956032, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 408318728192, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 408318734336, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.52.attn_q_b.weight", "offset": 408334462976, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.52.attn_norm.weight", "offset": 408409960448, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 408409980928, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 410926563328, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 413443145728, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.52.ffn_norm.weight", "offset": 415959728128, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 415959748608, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 415963025408, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 415994482688, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 416025939968, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 416057397248, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 416057399296, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.53.attn_kv_b.weight", "offset": 416063297536, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.53.attn_output.weight", "offset": 416096851968, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 416264624128, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 416264630272, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.53.attn_q_b.weight", "offset": 416280358912, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.53.attn_norm.weight", "offset": 416355856384, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 416355876864, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 418872459264, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 421389041664, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.53.ffn_norm.weight", "offset": 423905624064, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 423905644544, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 423908921344, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 423940378624, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 423971835904, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 424003293184, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 424003295232, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.54.attn_kv_b.weight", "offset": 424009193472, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.54.attn_output.weight", "offset": 424042747904, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 424210520064, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 424210526208, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.54.attn_q_b.weight", "offset": 424226254848, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.54.attn_norm.weight", "offset": 424301752320, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 424301772800, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 426818355200, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 429334937600, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.54.ffn_norm.weight", "offset": 431851520000, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 431851540480, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 431854817280, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 431886274560, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 431917731840, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 431949189120, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 431949191168, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.55.attn_kv_b.weight", "offset": 431955089408, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.55.attn_output.weight", "offset": 431988643840, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 432156416000, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 432156422144, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.55.attn_q_b.weight", "offset": 432172150784, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.55.attn_norm.weight", "offset": 432247648256, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 432247668736, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 434764251136, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 437280833536, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.55.ffn_norm.weight", "offset": 439797415936, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 439797436416, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 439800713216, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 439832170496, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 439863627776, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 439895085056, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 439895087104, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.56.attn_kv_b.weight", "offset": 439900985344, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.56.attn_output.weight", "offset": 439934539776, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 440102311936, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 440102318080, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.56.attn_q_b.weight", "offset": 440118046720, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.56.attn_norm.weight", "offset": 440193544192, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 440193564672, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 442710147072, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 445226729472, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.56.ffn_norm.weight", "offset": 447743311872, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 447743332352, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 447743352832, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 450259935232, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 452776517632, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 455293100032, "shape": [ 5120, 160 ], "size": 2516582400, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 455296376832, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 455327834112, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 455359291392, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.57.ffn_norm.weight", "offset": 455390748672, "shape": [ 5120 ], "size": 31457280, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 455390769152, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 455390771200, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.57.attn_kv_b.weight", "offset": 455396669440, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.57.attn_output.weight", "offset": 455430223872, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 455597996032, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 455598002176, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.57.attn_q_b.weight", "offset": 455613730816, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 455689228288, "shape": [ 5120, 160 ], "size": 75497472, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 455692505088, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 455723962368, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 455755419648, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 455786876928, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 455786878976, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.58.attn_kv_b.weight", "offset": 455792777216, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.58.attn_output.weight", "offset": 455826331648, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 455994103808, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 455994109952, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.58.attn_q_b.weight", "offset": 456009838592, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.58.attn_norm.weight", "offset": 456085336064, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 456085356544, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 458601938944, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 461118521344, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.58.ffn_norm.weight", "offset": 463635103744, "shape": [ 5120 ], "size": 2516582400, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 463635124224, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 463638401024, "shape": [ 3072, 5120 ], "size": 3276800, "type": 1 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 463669858304, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 463701315584, "shape": [ 5120, 3072 ], "size": 31457280, "type": 1 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 463732772864, "shape": [ 512 ], "size": 31457280, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 463732774912, "shape": [ 5120, 576 ], "size": 2048, "type": 1 }, { "name": "blk.59.attn_kv_b.weight", "offset": 463738673152, "shape": [ 512, 32768 ], "size": 5898240, "type": 1 }, { "name": "blk.59.attn_output.weight", "offset": 463772227584, "shape": [ 16384, 5120 ], "size": 33554432, "type": 1 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 463939999744, "shape": [ 1536 ], "size": 167772160, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 463940005888, "shape": [ 5120, 1536 ], "size": 6144, "type": 1 }, { "name": "blk.59.attn_q_b.weight", "offset": 463955734528, "shape": [ 1536, 24576 ], "size": 15728640, "type": 1 }, { "name": "blk.59.attn_norm.weight", "offset": 464031232000, "shape": [ 5120 ], "size": 75497472, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 464031252480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 1 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 466547834880, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 469064417280, "shape": [ 5120, 1536, 160 ], "size": 2516582400, "type": 1 }, { "name": "blk.59.ffn_norm.weight", "offset": 471580999680, "shape": [ 5120 ], "size": 2516582400, "type": 0 } ], "version": 3 }