deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 10, "general.name": "DeepSeek-Coder-V2-Lite-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 10 }, { "name": "blk.0.attn_norm.weight", "offset": 240844800, "shape": [ 2048 ], "size": 68812800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 240852992, "shape": [ 10944, 2048 ], "size": 8192, "type": 20 }, { "name": "blk.0.ffn_gate.weight", "offset": 253460480, "shape": [ 2048, 10944 ], "size": 12607488, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 260814848, "shape": [ 2048, 10944 ], "size": 7354368, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 268169216, "shape": [ 2048 ], "size": 7354368, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 268177408, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 268179456, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.0.attn_kv_b.weight", "offset": 268566528, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 269254656, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.0.attn_q.weight", "offset": 271056896, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.1.attn_norm.weight", "offset": 273121280, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 273129472, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 376938496, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 437493760, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 498049024, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 498573312, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 501051392, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 502943744, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 504836096, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 504844288, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 504846336, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.1.attn_kv_b.weight", "offset": 505233408, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 505921536, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.1.attn_q.weight", "offset": 507723776, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.2.attn_norm.weight", "offset": 509788160, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 509796352, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 613605376, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 674160640, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 734715904, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 735240192, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 737718272, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 739610624, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 741502976, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 741511168, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 741513216, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.2.attn_kv_b.weight", "offset": 741900288, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 742588416, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.2.attn_q.weight", "offset": 744390656, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.3.attn_norm.weight", "offset": 746455040, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 746463232, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 850272256, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 910827520, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 971382784, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 971907072, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 974385152, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 976277504, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 978169856, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 978178048, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 978180096, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.3.attn_kv_b.weight", "offset": 978567168, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 979255296, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.3.attn_q.weight", "offset": 981057536, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.4.attn_norm.weight", "offset": 983121920, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 983130112, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1086939136, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1147494400, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 1208049664, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 1208573952, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 1211052032, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 1212944384, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 1214836736, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 1214844928, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 1214846976, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.4.attn_kv_b.weight", "offset": 1215234048, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 1215922176, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.4.attn_q.weight", "offset": 1217724416, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.5.attn_norm.weight", "offset": 1219788800, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 1219796992, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 1323606016, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 1384161280, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 1444716544, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 1445240832, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 1447718912, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 1449611264, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 1451503616, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 1451511808, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 1451513856, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.5.attn_kv_b.weight", "offset": 1451900928, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 1452589056, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.5.attn_q.weight", "offset": 1454391296, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.6.attn_norm.weight", "offset": 1456455680, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 1456463872, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 1560272896, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 1620828160, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 1681383424, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 1681907712, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 1684385792, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 1686278144, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.6.ffn_norm.weight", "offset": 1688170496, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 1688178688, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 1688180736, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.6.attn_kv_b.weight", "offset": 1688567808, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 1689255936, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.6.attn_q.weight", "offset": 1691058176, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 1693122560, "shape": [ 2048, 64 ], "size": 2064384, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 1693646848, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 1696124928, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 1698017280, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 1699909632, "shape": [ 512 ], "size": 1892352, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 1699911680, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.7.attn_kv_b.weight", "offset": 1700298752, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 1700986880, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.7.attn_q.weight", "offset": 1702789120, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "output_norm.weight", "offset": 1704853504, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 1704861696, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 1704869888, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 1808678912, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 1869234176, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 1929789440, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 1930313728, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 1932791808, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 1934684160, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 1936576512, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 1936584704, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 1936586752, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.10.attn_kv_b.weight", "offset": 1936973824, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 1937661952, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 1939464192, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.11.attn_norm.weight", "offset": 1941528576, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 1941536768, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 2045345792, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 2105901056, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 2166456320, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 2166980608, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 2169458688, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 2171351040, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 2173243392, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 2173251584, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 2173253632, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.11.attn_kv_b.weight", "offset": 2173640704, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 2174328832, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.11.attn_q.weight", "offset": 2176131072, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.12.attn_norm.weight", "offset": 2178195456, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 2178203648, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 2282012672, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 2342567936, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 2403123200, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 2403647488, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 2406125568, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 2408017920, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 2409910272, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 2409918464, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 2409920512, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.12.attn_kv_b.weight", "offset": 2410307584, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 2410995712, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.12.attn_q.weight", "offset": 2412797952, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.13.attn_norm.weight", "offset": 2414862336, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 2414870528, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 2518679552, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 2579234816, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 2639790080, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 2640314368, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 2642792448, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 2644684800, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 2646577152, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 2646585344, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 2646587392, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.13.attn_kv_b.weight", "offset": 2646974464, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 2647662592, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 2649464832, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 2651529216, "shape": [ 2048, 64 ], "size": 2064384, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 2652053504, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 2654531584, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 2656423936, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 2658316288, "shape": [ 512 ], "size": 1892352, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 2658318336, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.14.attn_kv_b.weight", "offset": 2658705408, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 2659393536, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.14.attn_q.weight", "offset": 2661195776, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.7.attn_norm.weight", "offset": 2663260160, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 2663268352, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 2767077376, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 2827632640, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 2888187904, "shape": [ 2048 ], "size": 60555264, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 2888196096, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 2888204288, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 2992013312, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 3052568576, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 3113123840, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 3113648128, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 3116126208, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 3118018560, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 3119910912, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 3119919104, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 3119921152, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.8.attn_kv_b.weight", "offset": 3120308224, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 3120996352, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.8.attn_q.weight", "offset": 3122798592, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.9.attn_norm.weight", "offset": 3124862976, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 3124871168, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 3228680192, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 3289235456, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 3349790720, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 3350315008, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 3352793088, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 3354685440, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 3356577792, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 3356585984, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 3356588032, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.9.attn_kv_b.weight", "offset": 3356975104, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 3357663232, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.9.attn_q.weight", "offset": 3359465472, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.14.attn_norm.weight", "offset": 3361529856, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 3361538048, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 3465347072, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 3525902336, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 3586457600, "shape": [ 2048 ], "size": 60555264, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 3586465792, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 3586473984, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 3690283008, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 3750838272, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 3811393536, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 3811917824, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 3814395904, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 3816288256, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 3818180608, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 3818188800, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 3818190848, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.15.attn_kv_b.weight", "offset": 3818577920, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 3819266048, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.15.attn_q.weight", "offset": 3821068288, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.16.attn_norm.weight", "offset": 3823132672, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 3823140864, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 3926949888, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 3987505152, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 4048060416, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 4048584704, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 4051062784, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 4052955136, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 4054847488, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 4054855680, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 4054857728, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.16.attn_kv_b.weight", "offset": 4055244800, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 4055932928, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.16.attn_q.weight", "offset": 4057735168, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.17.attn_norm.weight", "offset": 4059799552, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 4059807744, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 4163616768, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 4224172032, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 4284727296, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 4285251584, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 4287729664, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 4289622016, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 4291514368, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 4291522560, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 4291524608, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.17.attn_kv_b.weight", "offset": 4291911680, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 4292599808, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.17.attn_q.weight", "offset": 4294402048, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.18.attn_norm.weight", "offset": 4296466432, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 4296474624, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 4400283648, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 4460838912, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 4521394176, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 4521918464, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 4524396544, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 4526288896, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 4528181248, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 4528189440, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 4528191488, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.18.attn_kv_b.weight", "offset": 4528578560, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 4529266688, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.18.attn_q.weight", "offset": 4531068928, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.19.attn_norm.weight", "offset": 4533133312, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 4533141504, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 4636950528, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 4697505792, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 4758061056, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 4758585344, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 4761063424, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 4762955776, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 4764848128, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 4764856320, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "\ufffd\u003c\u001d\ufffdй\u000fm5\u001bƍ\ufffd\ufffd\ufffdW\ufffdBk\u003e\ufffdh\u0011XwK(", "offset": 4764858368, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.19.attn_kv_b.weight", "offset": 4765245440, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 4765933568, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.19.attn_q.weight", "offset": 4767735808, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.20.attn_norm.weight", "offset": 4769800192, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 4769808384, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 4873617408, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 4934172672, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 4994727936, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 4995252224, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 4997730304, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 4999622656, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 5001515008, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 5001523200, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 5001525248, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.20.attn_kv_b.weight", "offset": 5001912320, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 5002600448, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.20.attn_q.weight", "offset": 5004402688, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.21.attn_norm.weight", "offset": 5006467072, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 5006475264, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 5110284288, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 5170839552, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 5231394816, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 5231919104, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 5234397184, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 5236289536, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 5238181888, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 5238190080, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 5238192128, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.21.attn_kv_b.weight", "offset": 5238579200, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 5239267328, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.21.attn_q.weight", "offset": 5241069568, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 5243133952, "shape": [ 2048, 64 ], "size": 2064384, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 5243658240, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 5246136320, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 5248028672, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 5249921024, "shape": [ 512 ], "size": 1892352, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 5249923072, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.22.attn_kv_b.weight", "offset": 5250310144, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 5250998272, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.22.attn_q.weight", "offset": 5252800512, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.22.attn_norm.weight", "offset": 5254864896, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 5254873088, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 5358682112, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 5419237376, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 5479792640, "shape": [ 2048 ], "size": 60555264, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 5479800832, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 5479809024, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 5583618048, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 5644173312, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 5704728576, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 5705252864, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 5707730944, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 5709623296, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 5711515648, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 5711523840, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 5711525888, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.23.attn_kv_b.weight", "offset": 5711912960, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 5712601088, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.23.attn_q.weight", "offset": 5714403328, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.24.attn_norm.weight", "offset": 5716467712, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 5716475904, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 5820284928, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 5880840192, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 5941395456, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 5941919744, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 5944397824, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 5946290176, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 5948182528, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 5948190720, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 5948192768, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.24.attn_kv_b.weight", "offset": 5948579840, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 5949267968, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.24.attn_q.weight", "offset": 5951070208, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.25.attn_norm.weight", "offset": 5953134592, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 5953142784, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 6056951808, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 6117507072, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 6178062336, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 6178586624, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 6181064704, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 6182957056, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 6184849408, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 6184857600, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 6184859648, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.25.attn_kv_b.weight", "offset": 6185246720, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 6185934848, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.25.attn_q.weight", "offset": 6187737088, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 }, { "name": "blk.26.attn_norm.weight", "offset": 6189801472, "shape": [ 2048 ], "size": 2064384, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 6189809664, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 20 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 6293618688, "shape": [ 2048, 1408, 64 ], "size": 103809024, "type": 10 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 6354173952, "shape": [ 2048, 1408, 64 ], "size": 60555264, "type": 10 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 6414729216, "shape": [ 2048, 64 ], "size": 60555264, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 6415253504, "shape": [ 2816, 2048 ], "size": 524288, "type": 11 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 6417731584, "shape": [ 2048, 2816 ], "size": 2478080, "type": 10 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 6419623936, "shape": [ 2048, 2816 ], "size": 1892352, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 6421516288, "shape": [ 2048 ], "size": 1892352, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 6421524480, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 6421526528, "shape": [ 2048, 576 ], "size": 2048, "type": 10 }, { "name": "blk.26.attn_kv_b.weight", "offset": 6421913600, "shape": [ 512, 4096 ], "size": 387072, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 6422601728, "shape": [ 2048, 2048 ], "size": 688128, "type": 11 }, { "name": "blk.26.attn_q.weight", "offset": 6424403968, "shape": [ 2048, 3072 ], "size": 1802240, "type": 10 } ], "version": 3 }