deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 13, "general.name": "DeepSeek-Coder-V2-Lite-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 262144000, "shape": [ 2048 ], "size": 90112000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 262152192, "shape": [ 10944, 2048 ], "size": 8192, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 278962176, "shape": [ 2048, 10944 ], "size": 16809984, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 288592896, "shape": [ 2048, 10944 ], "size": 9630720, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 298223616, "shape": [ 2048 ], "size": 9630720, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 298231808, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 298233856, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_kv_b.weight", "offset": 298740736, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 299641856, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 302525440, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 305228800, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 305236992, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 443649024, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 522947584, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 602246144, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 602770432, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 606735360, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 609213440, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 611691520, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 611699712, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 611701760, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_kv_b.weight", "offset": 612208640, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 613109760, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 615993344, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 618696704, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 618704896, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 757116928, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 836415488, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 915714048, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 916238336, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 920203264, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 922681344, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 925159424, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 925167616, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 925169664, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_kv_b.weight", "offset": 925676544, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 926577664, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 929461248, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 932164608, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 932172800, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1070584832, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1149883392, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1229181952, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1229706240, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1233671168, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1236149248, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 1238627328, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1238635520, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1238637568, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1239144448, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 1240045568, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 1242929152, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 1245632512, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1245640704, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1384052736, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1463351296, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 1542649856, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 1543174144, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 1547139072, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 1549617152, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 1552095232, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 1552103424, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 1552105472, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_kv_b.weight", "offset": 1552612352, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 1553513472, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 1556397056, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 1559100416, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 1559108608, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 1697520640, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 1776819200, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 1856117760, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 1856642048, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 1860606976, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 1863085056, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1865563136, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 1865571328, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 1865573376, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_kv_b.weight", "offset": 1866080256, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1866981376, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 1869864960, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1872568320, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 1872576512, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 2010988544, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 2090287104, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 2169585664, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 2170109952, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 2174074880, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 2176552960, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 2179031040, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 2179039232, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 2179041280, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_kv_b.weight", "offset": 2179548160, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 2180449280, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 2183332864, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 2186036224, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 2186560512, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 2190525440, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 2193003520, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 2195481600, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 2195483648, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_kv_b.weight", "offset": 2195990528, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 2196891648, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 2199775232, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "output_norm.weight", "offset": 2202478592, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 2202486784, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 2202494976, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 2340907008, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 2420205568, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 2499504128, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 2500028416, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 2503993344, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 2506471424, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 2508949504, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 2508957696, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 2508959744, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_kv_b.weight", "offset": 2509466624, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 2510367744, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 2513251328, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 2515954688, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 2515962880, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 2654374912, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 2733673472, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 2812972032, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 2813496320, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 2817461248, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 2819939328, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 2822417408, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 2822425600, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 2822427648, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_kv_b.weight", "offset": 2822934528, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 2823835648, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 2826719232, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 2829422592, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 2829430784, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 2967842816, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 3047141376, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 3126439936, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 3126964224, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 3130929152, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 3133407232, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 3135885312, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 3135893504, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 3135895552, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_kv_b.weight", "offset": 3136402432, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 3137303552, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 3140187136, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 3142890496, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 3142898688, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 3281310720, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 3360609280, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 3439907840, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 3440432128, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 3444397056, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 3446875136, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 3449353216, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 3449361408, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 3449363456, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_kv_b.weight", "offset": 3449870336, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 3450771456, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 3453655040, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 3456358400, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 3456882688, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 3460847616, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 3463325696, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 3465803776, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 3465805824, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_kv_b.weight", "offset": 3466312704, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 3467213824, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 3470097408, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 3472800768, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 3472808960, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 3611220992, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 3690519552, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 3769818112, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 3769826304, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 3769834496, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 3908246528, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 3987545088, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 4066843648, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 4067367936, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 4071332864, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 4073810944, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 4076289024, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 4076297216, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 4076299264, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_kv_b.weight", "offset": 4076806144, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 4077707264, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 4080590848, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 4083294208, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 4083302400, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 4221714432, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 4301012992, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 4380311552, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 4380835840, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 4384800768, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 4387278848, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 4389756928, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 4389765120, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 4389767168, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_kv_b.weight", "offset": 4390274048, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 4391175168, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 4394058752, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 4396762112, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 4396770304, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 4535182336, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 4614480896, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 4693779456, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 4693787648, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 4693795840, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 4832207872, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 4911506432, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 4990804992, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 4991329280, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 4995294208, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 4997772288, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 5000250368, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 5000258560, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 5000260608, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_kv_b.weight", "offset": 5000767488, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 5001668608, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 5004552192, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 5007255552, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 5007263744, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 5145675776, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 5224974336, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 5304272896, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 5304797184, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 5308762112, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 5311240192, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 5313718272, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 5313726464, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 5313728512, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_kv_b.weight", "offset": 5314235392, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 5315136512, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 5318020096, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 5320723456, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 5320731648, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 5459143680, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 5538442240, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 5617740800, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 5618265088, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 5622230016, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 5624708096, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 5627186176, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 5627194368, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 5627196416, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_kv_b.weight", "offset": 5627703296, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 5628604416, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 5631488000, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 5634191360, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 5634199552, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 5772611584, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 5851910144, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 5931208704, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 5931732992, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 5935697920, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 5938176000, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 5940654080, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 5940662272, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 5940664320, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_kv_b.weight", "offset": 5941171200, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 5942072320, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 5944955904, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 5947659264, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 5947667456, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 6086079488, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 6165378048, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 6244676608, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 6245200896, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 6249165824, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 6251643904, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 6254121984, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 6254130176, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 6254132224, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_kv_b.weight", "offset": 6254639104, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 6255540224, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 6258423808, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 6261127168, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 6261135360, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 6399547392, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 6478845952, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 6558144512, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 6558668800, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 6562633728, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 6565111808, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 6567589888, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 6567598080, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 6567600128, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_kv_b.weight", "offset": 6568107008, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 6569008128, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 6571891712, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 6574595072, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 6574603264, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 6713015296, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 6792313856, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 6871612416, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 6872136704, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 6876101632, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 6878579712, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 6881057792, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 6881065984, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 6881068032, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_kv_b.weight", "offset": 6881574912, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 6882476032, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 6885359616, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 6888062976, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 6888587264, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 6892552192, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 6895030272, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 6897508352, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 6897510400, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_kv_b.weight", "offset": 6898017280, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 6898918400, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 6901801984, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 6904505344, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 6904513536, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 7042925568, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 7122224128, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 7201522688, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 7201530880, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 7201539072, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 7339951104, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 7419249664, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 7498548224, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 7499072512, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 7503037440, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 7505515520, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 7507993600, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 7508001792, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 7508003840, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_kv_b.weight", "offset": 7508510720, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 7509411840, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 7512295424, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 7514998784, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 7515006976, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 7653419008, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 7732717568, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 7812016128, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 7812540416, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 7816505344, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 7818983424, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 7821461504, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 7821469696, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 7821471744, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_kv_b.weight", "offset": 7821978624, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 7822879744, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 7825763328, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 7828466688, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 7828474880, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 7966886912, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 8046185472, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 8125484032, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 8126008320, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 8129973248, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 8132451328, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 8134929408, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 8134937600, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 8134939648, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_kv_b.weight", "offset": 8135446528, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 8136347648, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 8139231232, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 8141934592, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 8141942784, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 8280354816, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 8359653376, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 8438951936, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 8439476224, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 8443441152, "shape": [ 2048, 2816 ], "size": 3964928, "type": 11 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 8445919232, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 8448397312, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 8448405504, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 8448407552, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_kv_b.weight", "offset": 8448914432, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 8449815552, "shape": [ 2048, 2048 ], "size": 901120, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 8452699136, "shape": [ 2048, 3072 ], "size": 2883584, "type": 11 } ], "version": 3 }