deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 12, "general.name": "DeepSeek-Coder-V2-Lite-Base", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 262144000, "shape": [ 2048 ], "size": 90112000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 262152192, "shape": [ 10944, 2048 ], "size": 8192, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 278962176, "shape": [ 2048, 10944 ], "size": 16809984, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 288592896, "shape": [ 2048, 10944 ], "size": 9630720, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 298223616, "shape": [ 2048 ], "size": 9630720, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 298231808, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 298233856, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_kv_b.weight", "offset": 298740736, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 299641856, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 302001152, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 304704512, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 304712704, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 431590400, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 510888960, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 590187520, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 590711808, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 593955840, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 596433920, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 598912000, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 598920192, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 598922240, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_kv_b.weight", "offset": 599429120, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 600330240, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 602689536, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 605392896, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 605401088, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 732278784, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 811577344, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 890875904, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 891400192, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 894644224, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 897122304, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 899600384, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 899608576, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 899610624, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_kv_b.weight", "offset": 900117504, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 901018624, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 903377920, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 906081280, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 906089472, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1032967168, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1112265728, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1191564288, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1192088576, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1195332608, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1197810688, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 1200288768, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1200296960, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1200299008, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1200805888, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 1201707008, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 1204066304, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 1206769664, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1206777856, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1333655552, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1412954112, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 1492252672, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 1492776960, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 1496020992, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 1498499072, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 1500977152, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 1500985344, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 1500987392, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_kv_b.weight", "offset": 1501494272, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 1502395392, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 1504754688, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 1507458048, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 1507466240, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 1634343936, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 1713642496, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 1792941056, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 1793465344, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 1796709376, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 1799187456, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1801665536, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 1801673728, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 1801675776, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_kv_b.weight", "offset": 1802182656, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 1803083776, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 1805443072, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1808146432, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 1808154624, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 1935032320, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 2014330880, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 2093629440, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 2094153728, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 2097397760, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 2099875840, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 2102353920, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 2102362112, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 2102364160, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_kv_b.weight", "offset": 2102871040, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 2103772160, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 2106131456, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 2108834816, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 2109359104, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 2112603136, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 2115081216, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 2117559296, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 2117561344, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_kv_b.weight", "offset": 2118068224, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 2118969344, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 2121328640, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "output_norm.weight", "offset": 2124032000, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 2124040192, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 2124048384, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 2250926080, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 2330224640, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 2409523200, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 2410047488, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 2413291520, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 2415769600, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 2418247680, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 2418255872, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 2418257920, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_kv_b.weight", "offset": 2418764800, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 2419665920, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 2422025216, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 2424728576, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 2424736768, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 2551614464, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 2630913024, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 2710211584, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 2710735872, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 2713979904, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 2716457984, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 2718936064, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 2718944256, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 2718946304, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_kv_b.weight", "offset": 2719453184, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 2720354304, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.11.attn_q.weight", "offset": 2722713600, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 2725416960, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 2725425152, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 2852302848, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 2931601408, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 3010899968, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 3011424256, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 3014668288, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 3017146368, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 3019624448, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 3019632640, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 3019634688, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_kv_b.weight", "offset": 3020141568, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 3021042688, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 3023401984, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 3026105344, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 3026113536, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 3152991232, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 3232289792, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 3311588352, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 3312112640, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 3315356672, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 3317834752, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 3320312832, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 3320321024, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 3320323072, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_kv_b.weight", "offset": 3320829952, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 3321731072, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 3324090368, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 3326793728, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 3327318016, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 3330562048, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 3333040128, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 3335518208, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 3335520256, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_kv_b.weight", "offset": 3336027136, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 3336928256, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 3339287552, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 3341990912, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 3341999104, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 3468876800, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 3548175360, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 3627473920, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 3627482112, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 3627490304, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 3754368000, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 3833666560, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 3912965120, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 3913489408, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 3916733440, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 3919211520, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 3921689600, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 3921697792, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 3921699840, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_kv_b.weight", "offset": 3922206720, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 3923107840, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 3925467136, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 3928170496, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 3928178688, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 4055056384, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 4134354944, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 4213653504, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 4214177792, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 4217421824, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 4219899904, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 4222377984, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 4222386176, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 4222388224, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_kv_b.weight", "offset": 4222895104, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 4223796224, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 4226155520, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 4228858880, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 4228867072, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 4355744768, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 4435043328, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 4514341888, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 4514350080, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 4514358272, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 4641235968, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 4720534528, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 4799833088, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 4800357376, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 4803601408, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 4806079488, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 4808557568, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 4808565760, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 4808567808, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_kv_b.weight", "offset": 4809074688, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 4809975808, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 4812335104, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 4815038464, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 4815046656, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 4941924352, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 5021222912, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 5100521472, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 5101045760, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 5104289792, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 5106767872, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 5109245952, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 5109254144, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 5109256192, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_kv_b.weight", "offset": 5109763072, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 5110664192, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.16.attn_q.weight", "offset": 5113023488, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 5115726848, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 5115735040, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 5242612736, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 5321911296, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 5401209856, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 5401734144, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 5404978176, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 5407456256, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 5409934336, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 5409942528, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 5409944576, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_kv_b.weight", "offset": 5410451456, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 5411352576, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 5413711872, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 5416415232, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 5416423424, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 5543301120, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 5622599680, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 5701898240, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 5702422528, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 5705666560, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 5708144640, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 5710622720, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 5710630912, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 5710632960, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_kv_b.weight", "offset": 5711139840, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 5712040960, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.18.attn_q.weight", "offset": 5714400256, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 5717103616, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 5717111808, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 5843989504, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 5923288064, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 6002586624, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 6003110912, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 6006354944, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 6008833024, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 6011311104, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 6011319296, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 6011321344, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_kv_b.weight", "offset": 6011828224, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 6012729344, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 6015088640, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 6017792000, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 6017800192, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 6144677888, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 6223976448, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 6303275008, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 6303799296, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 6307043328, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 6309521408, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 6311999488, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 6312007680, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 6312009728, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_kv_b.weight", "offset": 6312516608, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 6313417728, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 6315777024, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 6318480384, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 6318488576, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 6445366272, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 6524664832, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 6603963392, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 6604487680, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 6607731712, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 6610209792, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 6612687872, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 6612696064, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 6612698112, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_kv_b.weight", "offset": 6613204992, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 6614106112, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 6616465408, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 6619168768, "shape": [ 2048, 64 ], "size": 2703360, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 6619693056, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 6622937088, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 6625415168, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 6627893248, "shape": [ 512 ], "size": 2478080, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 6627895296, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_kv_b.weight", "offset": 6628402176, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 6629303296, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 6631662592, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 6634365952, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 6634374144, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 6761251840, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 6840550400, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 6919848960, "shape": [ 2048 ], "size": 79298560, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 6919857152, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 6919865344, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 7046743040, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 7126041600, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 7205340160, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 7205864448, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 7209108480, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 7211586560, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 7214064640, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 7214072832, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 7214074880, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_kv_b.weight", "offset": 7214581760, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 7215482880, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 7217842176, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 7220545536, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 7220553728, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 7347431424, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 7426729984, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 7506028544, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 7506552832, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 7509796864, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 7512274944, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 7514753024, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 7514761216, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 7514763264, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_kv_b.weight", "offset": 7515270144, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 7516171264, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 7518530560, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 7521233920, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 7521242112, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 7648119808, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 7727418368, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 7806716928, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 7807241216, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 7810485248, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 7812963328, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 7815441408, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 7815449600, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 7815451648, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_kv_b.weight", "offset": 7815958528, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 7816859648, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 7819218944, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 7821922304, "shape": [ 2048 ], "size": 2703360, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 7821930496, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 7948808192, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 11 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 8028106752, "shape": [ 2048, 1408, 64 ], "size": 79298560, "type": 11 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 8107405312, "shape": [ 2048, 64 ], "size": 79298560, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 8107929600, "shape": [ 2816, 2048 ], "size": 524288, "type": 12 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 8111173632, "shape": [ 2048, 2816 ], "size": 3244032, "type": 11 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 8113651712, "shape": [ 2048, 2816 ], "size": 2478080, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 8116129792, "shape": [ 2048 ], "size": 2478080, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 8116137984, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 8116140032, "shape": [ 2048, 576 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_kv_b.weight", "offset": 8116646912, "shape": [ 512, 4096 ], "size": 506880, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 8117548032, "shape": [ 2048, 2048 ], "size": 901120, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 8119907328, "shape": [ 2048, 3072 ], "size": 2359296, "type": 11 } ], "version": 3 }