deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 17, "general.name": "DeepSeek-Coder-V2-Lite-Base", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 316211200, "shape": [ 2048 ], "size": 144179200, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 316219392, "shape": [ 10944, 2048 ], "size": 8192, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 340033536, "shape": [ 2048, 10944 ], "size": 23814144, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 355442688, "shape": [ 2048, 10944 ], "size": 15409152, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 370851840, "shape": [ 2048 ], "size": 15409152, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 370860032, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 370862080, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.0.attn_kv_b.weight", "offset": 371673088, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 373114880, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 375998464, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 380323840, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 380332032, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 576415744, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 703293440, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 830171136, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 830695424, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 835426304, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 839391232, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 843356160, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 843364352, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 843366400, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_kv_b.weight", "offset": 844177408, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 845619200, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 848502784, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 852828160, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 852836352, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 1048920064, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 1175797760, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1302675456, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 1303199744, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 1307930624, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 1311895552, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 1315860480, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 1315868672, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 1315870720, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_kv_b.weight", "offset": 1316681728, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 1318123520, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 1321007104, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 1325332480, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 1325340672, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1463752704, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1590630400, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1717508096, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1718032384, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1721997312, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1725962240, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 1729927168, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1729935360, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1729937408, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1730748416, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 1732190208, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 1735073792, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1739399168, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1739407360, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1877819392, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 2004697088, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 2131574784, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 2132099072, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 2136064000, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 2140028928, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 2143993856, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 2144002048, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 2144004096, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_kv_b.weight", "offset": 2144815104, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 2146256896, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 2149140480, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 2153465856, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 2153474048, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 2349557760, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 2476435456, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 2603313152, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 2603837440, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 2608568320, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 2612533248, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 2616498176, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 2616506368, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 2616508416, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.5.attn_kv_b.weight", "offset": 2617319424, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 2618761216, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 2621644800, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 2625970176, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 2625978368, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 2764390400, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 2891268096, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "\u0000\u0000\u0000\u0000\b\u0000\u0000\u0000\u0000\u0000\u0000\r\u0000\u0000\u0000\u0000@\ufffd\ufffd\u0000\u0000\u0000\u0000\u001b\u0000", "offset": 3018145792, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 3018670080, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 3022635008, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 3026599936, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 3030564864, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 3030573056, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 3030575104, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.6.attn_kv_b.weight", "offset": 3031386112, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 3032827904, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 3035711488, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 3040036864, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 3040561152, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 3044526080, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 3048491008, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 3052455936, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 3052457984, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.7.attn_kv_b.weight", "offset": 3053268992, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 3054710784, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 3057594368, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "output_norm.weight", "offset": 3061919744, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 3061927936, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 3061936128, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 3200348160, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 3327225856, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 3454103552, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 3454627840, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 3458592768, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 3462557696, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 3466522624, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 3466530816, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 3466532864, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.10.attn_kv_b.weight", "offset": 3467343872, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 3468785664, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 3471669248, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 3475994624, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 3476002816, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 3672086528, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 3798964224, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 3925841920, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 3926366208, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 3931097088, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 3935062016, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 3939026944, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 3939035136, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 3939037184, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.11.attn_kv_b.weight", "offset": 3939848192, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 3941289984, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 3944173568, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 3948498944, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 3948507136, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 4086919168, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 4213796864, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 4340674560, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 4341198848, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 4345163776, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 4349128704, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 4353093632, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 4353101824, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 4353103872, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.12.attn_kv_b.weight", "offset": 4353914880, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 4355356672, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 4358240256, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 4362565632, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 4362573824, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 4500985856, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 4627863552, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 4754741248, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 4755265536, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 4759230464, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 4763195392, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 4767160320, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 4767168512, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 4767170560, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.13.attn_kv_b.weight", "offset": 4767981568, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 4769423360, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 4772306944, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 4776632320, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 4777156608, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 4781887488, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 4785852416, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 4789817344, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 4789819392, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.14.attn_kv_b.weight", "offset": 4790630400, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 4792072192, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 4794955776, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 4799281152, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 4799289344, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 4937701376, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 5064579072, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 5191456768, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5191464960, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 5191473152, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 5387556864, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 5514434560, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 5641312256, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 5641836544, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 5646567424, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 5650532352, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 5654497280, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 5654505472, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 5654507520, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.8.attn_kv_b.weight", "offset": 5655318528, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 5656760320, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 5659643904, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 5663969280, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 5663977472, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 5802389504, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 5929267200, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 6056144896, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 6056669184, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 6060634112, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 6064599040, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 6068563968, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 6068572160, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 6068574208, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.9.attn_kv_b.weight", "offset": 6069385216, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 6070827008, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 6073710592, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 6078035968, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 6078044160, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 6274127872, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 6401005568, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 6527883264, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 6527891456, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 6527899648, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 6666311680, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 6793189376, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 6920067072, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 6920591360, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 6924556288, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 6928521216, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 6932486144, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 6932494336, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 6932496384, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.15.attn_kv_b.weight", "offset": 6933307392, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 6934749184, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 6937632768, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 6941958144, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 6941966336, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 7080378368, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 7207256064, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 7334133760, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 7334658048, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 7338622976, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 7342587904, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 7346552832, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 7346561024, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 7346563072, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.16.attn_kv_b.weight", "offset": 7347374080, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 7348815872, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 7351699456, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 7356024832, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 7356033024, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 7552116736, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 7678994432, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 7805872128, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 7806396416, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 7811127296, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 7815092224, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 7819057152, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 7819065344, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 7819067392, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.17.attn_kv_b.weight", "offset": 7819878400, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 7821320192, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 7824203776, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 7828529152, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 7828537344, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 7966949376, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 8093827072, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 8220704768, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 8221229056, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 8225193984, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 8229158912, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 8233123840, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 8233132032, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 8233134080, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.18.attn_kv_b.weight", "offset": 8233945088, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 8235386880, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 8238270464, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 8242595840, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 8242604032, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 8381016064, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 8507893760, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 8634771456, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 8635295744, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 8639260672, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 8643225600, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 8647190528, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 8647198720, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 8647200768, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.19.attn_kv_b.weight", "offset": 8648011776, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 8649453568, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 8652337152, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 8656662528, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 8656670720, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 8852754432, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 8979632128, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 9106509824, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 9107034112, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 9111764992, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 9115729920, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 9119694848, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 9119703040, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 9119705088, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.20.attn_kv_b.weight", "offset": 9120516096, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 9121957888, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 9124841472, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 9129166848, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 9129175040, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 9267587072, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 9394464768, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 9521342464, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 9521866752, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 9525831680, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 9529796608, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 9533761536, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 9533769728, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 9533771776, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.21.attn_kv_b.weight", "offset": 9534582784, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 9536024576, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 9538908160, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 9543233536, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 9543757824, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 9547722752, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 9551687680, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 9555652608, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 9555654656, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.22.attn_kv_b.weight", "offset": 9556465664, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 9557907456, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 9560791040, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 9565116416, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 9565124608, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 9703536640, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 9830414336, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 9957292032, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 9957300224, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 9957308416, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 10153392128, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 10280269824, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 10407147520, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 10407671808, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 10412402688, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 10416367616, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 10420332544, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 10420340736, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 10420342784, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.23.attn_kv_b.weight", "offset": 10421153792, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 10422595584, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 10425479168, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 10429804544, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 10429812736, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 10625896448, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 10752774144, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 10879651840, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 10880176128, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 10884907008, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 10888871936, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 10892836864, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 10892845056, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 10892847104, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.24.attn_kv_b.weight", "offset": 10893658112, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 10895099904, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 10897983488, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 10902308864, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 10902317056, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 11098400768, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 11225278464, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 11352156160, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 11352680448, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 11357411328, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 11361376256, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 11365341184, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 11365349376, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 11365351424, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.25.attn_kv_b.weight", "offset": 11366162432, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 11367604224, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 11370487808, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 11374813184, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 11374821376, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 11570905088, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 13 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 11697782784, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 11824660480, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 11825184768, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 11829915648, "shape": [ 2048, 2816 ], "size": 4730880, "type": 13 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 11833880576, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 11837845504, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 11837853696, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 11837855744, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.26.attn_kv_b.weight", "offset": 11838666752, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 11840108544, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 11842992128, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 } ], "version": 3 }