deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 11, "general.name": "DeepSeek-Coder-V2-Lite-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 262144000, "shape": [ 2048 ], "size": 90112000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 262152192, "shape": [ 10944, 2048 ], "size": 8192, "type": 20 }, { "name": "blk.0.ffn_gate.weight", "offset": 274759680, "shape": [ 2048, 10944 ], "size": 12607488, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 284390400, "shape": [ 2048, 10944 ], "size": 9630720, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 294021120, "shape": [ 2048 ], "size": 9630720, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 294029312, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 294031360, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_kv_b.weight", "offset": 294538240, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 295439360, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 297241600, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 299944960, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 299953152, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 403762176, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 483060736, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 562359296, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 562883584, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 565361664, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 567839744, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 570317824, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 570326016, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 570328064, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_kv_b.weight", "offset": 570834944, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 571736064, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 573538304, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 576241664, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 576249856, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 680058880, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 759357440, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 838656000, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 839180288, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 841658368, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 844136448, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 846614528, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 846622720, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 846624768, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_kv_b.weight", "offset": 847131648, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 848032768, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 849835008, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 852538368, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 852546560, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 956355584, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1035654144, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1114952704, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1115476992, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1117955072, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1120433152, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 1122911232, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1122919424, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1122921472, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1123428352, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 1124329472, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 1126131712, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 1128835072, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1128843264, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1232652288, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1311950848, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 1391249408, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 1391773696, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 1394251776, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 1396729856, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 1399207936, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 1399216128, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 1399218176, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_kv_b.weight", "offset": 1399725056, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 1400626176, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 1402428416, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 1405131776, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 1405139968, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 1508948992, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 1588247552, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 1667546112, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 1668070400, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 1670548480, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 1673026560, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1675504640, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 1675512832, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 1675514880, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_kv_b.weight", "offset": 1676021760, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1676922880, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 1678725120, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1681428480, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 1681436672, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 1785245696, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 1864544256, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 1943842816, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 1944367104, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 1946845184, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 1949323264, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1951801344, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 1951809536, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 1951811584, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_kv_b.weight", "offset": 1952318464, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1953219584, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 1955021824, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 1957725184, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 1958249472, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 1960727552, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 1963205632, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 1965683712, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 1965685760, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_kv_b.weight", "offset": 1966192640, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1967093760, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 1968896000, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "output_norm.weight", "offset": 1971599360, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 1971607552, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 1971615744, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 2075424768, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 2154723328, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 2234021888, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 2234546176, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 2237024256, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 2239502336, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 2241980416, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 2241988608, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 2241990656, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_kv_b.weight", "offset": 2242497536, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 2243398656, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 2245200896, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 2247904256, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 2247912448, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 2351721472, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 2431020032, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 2510318592, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 2510842880, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 2513320960, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 2515799040, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 2518277120, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 2518285312, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 2518287360, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_kv_b.weight", "offset": 2518794240, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 2519695360, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 2521497600, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 2524200960, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 2524209152, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 2628018176, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 2707316736, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 2786615296, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 2787139584, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 2789617664, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 2792095744, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 2794573824, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 2794582016, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 2794584064, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_kv_b.weight", "offset": 2795090944, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 2795992064, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 2797794304, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 2800497664, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 2800505856, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 2904314880, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 2983613440, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 3062912000, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 3063436288, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 3065914368, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 3068392448, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 3070870528, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 3070878720, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 3070880768, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_kv_b.weight", "offset": 3071387648, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 3072288768, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 3074091008, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 3076794368, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 3077318656, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 3079796736, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 3082274816, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 3084752896, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 3084754944, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_kv_b.weight", "offset": 3085261824, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 3086162944, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 3087965184, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 3090668544, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 3090676736, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 3194485760, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 3273784320, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 3353082880, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 3353091072, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 3353099264, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 3456908288, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 3536206848, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 3615505408, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 3616029696, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 3618507776, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 3620985856, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 3623463936, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 3623472128, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 3623474176, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_kv_b.weight", "offset": 3623981056, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 3624882176, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 3626684416, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 3629387776, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 3629395968, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 3733204992, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 3812503552, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 3891802112, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 3892326400, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 3894804480, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 3897282560, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 3899760640, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 3899768832, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 3899770880, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_kv_b.weight", "offset": 3900277760, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 3901178880, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 3902981120, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 3905684480, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 3905692672, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 4009501696, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 4088800256, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 4168098816, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 4168107008, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 4168115200, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 4271924224, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 4351222784, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 4430521344, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 4431045632, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 4433523712, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 4436001792, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 4438479872, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 4438488064, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 4438490112, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_kv_b.weight", "offset": 4438996992, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 4439898112, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 4441700352, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 4444403712, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 4444411904, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 4548220928, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 4627519488, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 4706818048, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 4707342336, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 4709820416, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 4712298496, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 4714776576, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 4714784768, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 4714786816, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_kv_b.weight", "offset": 4715293696, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 4716194816, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 4717997056, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 4720700416, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 4720708608, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 4824517632, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 4903816192, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 4983114752, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 4983639040, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 4986117120, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 4988595200, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 4991073280, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 4991081472, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 4991083520, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_kv_b.weight", "offset": 4991590400, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 4992491520, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 4994293760, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 4996997120, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 4997005312, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 5100814336, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 5180112896, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 5259411456, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 5259935744, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 5262413824, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 5264891904, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 5267369984, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 5267378176, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 5267380224, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_kv_b.weight", "offset": 5267887104, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 5268788224, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 5270590464, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 5273293824, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 5273302016, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 5377111040, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 5456409600, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 5535708160, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 5536232448, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 5538710528, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 5541188608, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 5543666688, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 5543674880, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 5543676928, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_kv_b.weight", "offset": 5544183808, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 5545084928, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 5546887168, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 5549590528, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 5549598720, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 5653407744, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 5732706304, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 5812004864, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 5812529152, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 5815007232, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 5817485312, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 5819963392, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 5819971584, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 5819973632, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_kv_b.weight", "offset": 5820480512, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 5821381632, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 5823183872, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 5825887232, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 5825895424, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 5929704448, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 6009003008, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 6088301568, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 6088825856, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 6091303936, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 6093782016, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 6096260096, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 6096268288, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 6096270336, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_kv_b.weight", "offset": 6096777216, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 6097678336, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 6099480576, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 6102183936, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 6102708224, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 6105186304, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 6107664384, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 6110142464, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 6110144512, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_kv_b.weight", "offset": 6110651392, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 6111552512, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 6113354752, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 6116058112, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 6116066304, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 6219875328, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 6299173888, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 6378472448, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 6378480640, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 6378488832, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 6482297856, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 6561596416, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 6640894976, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 6641419264, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 6643897344, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 6646375424, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 6648853504, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 6648861696, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 6648863744, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_kv_b.weight", "offset": 6649370624, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 6650271744, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 6652073984, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 6654777344, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 6654785536, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 6758594560, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 6837893120, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 6917191680, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 6917715968, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 6920194048, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 6922672128, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 6925150208, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 6925158400, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 6925160448, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_kv_b.weight", "offset": 6925667328, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 6926568448, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 6928370688, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 6931074048, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 6931082240, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 7034891264, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 7114189824, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 7193488384, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 7194012672, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 7196490752, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 7198968832, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 7201446912, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 7201455104, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 7201457152, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_kv_b.weight", "offset": 7201964032, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 7202865152, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 7204667392, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 7207370752, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 7207378944, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 7311187968, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 7390486528, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 7469785088, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 7470309376, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 7472787456, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 7475265536, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 7477743616, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 7477751808, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 7477753856, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_kv_b.weight", "offset": 7478260736, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 7479161856, "shape": [ 2048, 2048 ], "size": 901120, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 7480964096, "shape": [ 2048, 3072 ], "size": 1802240, "type": 11 } ], "version": 3 }