deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 16, "general.name": "DeepSeek-Coder-V2-Lite-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 316211200, "shape": [ 2048 ], "size": 144179200, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 316219392, "shape": [ 10944, 2048 ], "size": 8192, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 333029376, "shape": [ 2048, 10944 ], "size": 16809984, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 348438528, "shape": [ 2048, 10944 ], "size": 15409152, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 363847680, "shape": [ 2048 ], "size": 15409152, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 363855872, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 363857920, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.0.attn_kv_b.weight", "offset": 364668928, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 366110720, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 368994304, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 373319680, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 373327872, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 511739904, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 638617600, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 765495296, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 766019584, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 769984512, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 773949440, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 777914368, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 777922560, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 777924608, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_kv_b.weight", "offset": 778735616, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 780177408, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 783060992, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 787386368, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 787394560, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 925806592, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 1052684288, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1179561984, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 1180086272, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 1184051200, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 1188016128, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 1191981056, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 1191989248, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 1191991296, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_kv_b.weight", "offset": 1192802304, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 1194244096, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 1197127680, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 1201453056, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 1201461248, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1339873280, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1466750976, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1593628672, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1594152960, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1598117888, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1602082816, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 1606047744, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1606055936, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1606057984, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1606868992, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 1608310784, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 1611194368, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1615519744, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1615527936, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1753939968, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1880817664, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 2007695360, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 2008219648, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 2012184576, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 2016149504, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 2020114432, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 2020122624, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 2020124672, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_kv_b.weight", "offset": 2020935680, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 2022377472, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 2025261056, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 2029586432, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 2029594624, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 2168006656, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 2294884352, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 2421762048, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 2422286336, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 2426251264, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 2430216192, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 2434181120, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 2434189312, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 2434191360, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.5.attn_kv_b.weight", "offset": 2435002368, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 2436444160, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 2439327744, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 2443653120, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 2443661312, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 2582073344, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 2708951040, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "\u0000\u0000@\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0007\u0000\u0000\u0000\u0000\ufffd\ufffd\ufffd\u0000\u0000\u0000\u0000\u001b\u0000\u0000", "offset": 2835828736, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 2836353024, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 2840317952, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 2844282880, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 2848247808, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 2848256000, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 2848258048, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.6.attn_kv_b.weight", "offset": 2849069056, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 2850510848, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 2853394432, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 2857719808, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 2858244096, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 2862209024, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 2866173952, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 2870138880, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 2870140928, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.7.attn_kv_b.weight", "offset": 2870951936, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 2872393728, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 2875277312, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "output_norm.weight", "offset": 2879602688, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 2879610880, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 2879619072, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 3018031104, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 3144908800, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 3271786496, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 3272310784, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 3276275712, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 3280240640, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 3284205568, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 3284213760, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 3284215808, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.10.attn_kv_b.weight", "offset": 3285026816, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 3286468608, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 3289352192, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 3293677568, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 3293685760, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 3432097792, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 3558975488, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 3685853184, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 3686377472, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 3690342400, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 3694307328, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 3698272256, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 3698280448, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 3698282496, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.11.attn_kv_b.weight", "offset": 3699093504, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 3700535296, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.11.attn_q.weight", "offset": 3703418880, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 3707744256, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 3707752448, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 3846164480, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 3973042176, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 4099919872, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 4100444160, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 4104409088, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 4108374016, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 4112338944, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 4112347136, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 4112349184, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.12.attn_kv_b.weight", "offset": 4113160192, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 4114601984, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 4117485568, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 4121810944, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 4121819136, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 4260231168, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 4387108864, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 4513986560, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 4514510848, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 4518475776, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 4522440704, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 4526405632, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 4526413824, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 4526415872, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.13.attn_kv_b.weight", "offset": 4527226880, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 4528668672, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 4531552256, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 4535877632, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 4536401920, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 4540366848, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 4544331776, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 4548296704, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 4548298752, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.14.attn_kv_b.weight", "offset": 4549109760, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 4550551552, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 4553435136, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 4557760512, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 4557768704, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 4696180736, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 4823058432, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 4949936128, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 4949944320, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 4949952512, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 5088364544, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 5215242240, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 5342119936, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 5342644224, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 5346609152, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 5350574080, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 5354539008, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 5354547200, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 5354549248, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.8.attn_kv_b.weight", "offset": 5355360256, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 5356802048, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 5359685632, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 5364011008, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 5364019200, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 5502431232, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 5629308928, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 5756186624, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 5756710912, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 5760675840, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 5764640768, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 5768605696, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 5768613888, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 5768615936, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.9.attn_kv_b.weight", "offset": 5769426944, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 5770868736, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 5773752320, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 5778077696, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 5778085888, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 5916497920, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 6043375616, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 6170253312, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 6170261504, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 6170269696, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 6308681728, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "\b\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0005\u0000\u0000\u0000\u0000\u0000\u0000@\u0000\u0000\u0000\u0000\u0000\u0000\u0000\r\u0000", "offset": 6435559424, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 6562437120, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 6562961408, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 6566926336, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 6570891264, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 6574856192, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 6574864384, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 6574866432, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.15.attn_kv_b.weight", "offset": 6575677440, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 6577119232, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 6580002816, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 6584328192, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 6584336384, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 6722748416, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 6849626112, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 6976503808, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 6977028096, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 6980993024, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 6984957952, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 6988922880, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 6988931072, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 6988933120, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.16.attn_kv_b.weight", "offset": 6989744128, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 6991185920, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.16.attn_q.weight", "offset": 6994069504, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 6998394880, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 6998403072, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 7136815104, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 7263692800, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 7390570496, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 7391094784, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 7395059712, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 7399024640, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 7402989568, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 7402997760, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 7402999808, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.17.attn_kv_b.weight", "offset": 7403810816, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 7405252608, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 7408136192, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 7412461568, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 7412469760, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 7550881792, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 7677759488, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 7804637184, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 7805161472, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 7809126400, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 7813091328, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 7817056256, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 7817064448, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 7817066496, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.18.attn_kv_b.weight", "offset": 7817877504, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 7819319296, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.18.attn_q.weight", "offset": 7822202880, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 7826528256, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 7826536448, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 7964948480, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 8091826176, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 8218703872, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 8219228160, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 8223193088, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 8227158016, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 8231122944, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 8231131136, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 8231133184, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.19.attn_kv_b.weight", "offset": 8231944192, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 8233385984, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 8236269568, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 8240594944, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 8240603136, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 8379015168, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 8505892864, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 8632770560, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 8633294848, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 8637259776, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 8641224704, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 8645189632, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 8645197824, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 8645199872, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.20.attn_kv_b.weight", "offset": 8646010880, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 8647452672, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 8650336256, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 8654661632, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 8654669824, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 8793081856, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 8919959552, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 9046837248, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 9047361536, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 9051326464, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 9055291392, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 9059256320, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 9059264512, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 9059266560, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.21.attn_kv_b.weight", "offset": 9060077568, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 9061519360, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 9064402944, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 9068728320, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 9069252608, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 9073217536, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 9077182464, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 9081147392, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 9081149440, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.22.attn_kv_b.weight", "offset": 9081960448, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 9083402240, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 9086285824, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 9090611200, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 9090619392, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 9229031424, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 9355909120, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 9482786816, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 9482795008, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 9482803200, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 9621215232, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 9748092928, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 9874970624, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 9875494912, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 9879459840, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 9883424768, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 9887389696, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 9887397888, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 9887399936, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.23.attn_kv_b.weight", "offset": 9888210944, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 9889652736, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 9892536320, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 9896861696, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 9896869888, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 10035281920, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 10162159616, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 10289037312, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 10289561600, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 10293526528, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 10297491456, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 10301456384, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 10301464576, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 10301466624, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.24.attn_kv_b.weight", "offset": 10302277632, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 10303719424, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 10306603008, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 10310928384, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 10310936576, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 10449348608, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 10576226304, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 10703104000, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 10703628288, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 10707593216, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 10711558144, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 10715523072, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 10715531264, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 10715533312, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.25.attn_kv_b.weight", "offset": 10716344320, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 10717786112, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 10720669696, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 10724995072, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 10725003264, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 10863415296, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 13 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 10990292992, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 13 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 11117170688, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 11117694976, "shape": [ 2816, 2048 ], "size": 524288, "type": 13 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 11121659904, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 11125624832, "shape": [ 2048, 2816 ], "size": 3964928, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 11129589760, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 11129597952, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 11129600000, "shape": [ 2048, 576 ], "size": 2048, "type": 13 }, { "name": "blk.26.attn_kv_b.weight", "offset": 11130411008, "shape": [ 512, 4096 ], "size": 811008, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 11131852800, "shape": [ 2048, 2048 ], "size": 1441792, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 11134736384, "shape": [ 2048, 3072 ], "size": 2883584, "type": 13 } ], "version": 3 }