deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 18, "general.name": "DeepSeek-Coder-V2-Lite-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 14 }, { "name": "blk.0.attn_norm.weight", "offset": 344064000, "shape": [ 2048 ], "size": 172032000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 344072192, "shape": [ 10944, 2048 ], "size": 8192, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 367886336, "shape": [ 2048, 10944 ], "size": 23814144, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 386272256, "shape": [ 2048, 10944 ], "size": 18385920, "type": 14 }, { "name": "blk.0.ffn_norm.weight", "offset": 404658176, "shape": [ 2048 ], "size": 18385920, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 404666368, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 404668416, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.0.attn_kv_b.weight", "offset": 405636096, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 407356416, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.0.attn_q.weight", "offset": 410797056, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 415958016, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 415966208, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 612049920, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 763438080, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 914826240, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 915350528, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 920081408, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 924812288, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 929543168, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 929551360, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 929553408, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.1.attn_kv_b.weight", "offset": 930521088, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 932241408, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.1.attn_q.weight", "offset": 935682048, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 940843008, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 940851200, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 1136934912, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 1288323072, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1439711232, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 1440235520, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 1444966400, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 1449697280, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 1454428160, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 1454436352, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 1454438400, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.2.attn_kv_b.weight", "offset": 1455406080, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 1457126400, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.2.attn_q.weight", "offset": 1460567040, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 1465728000, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 1465736192, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1661819904, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1813208064, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1964596224, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1965120512, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1969851392, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1974582272, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 1979313152, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1979321344, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1979323392, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1980291072, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 1982011392, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.3.attn_q.weight", "offset": 1985452032, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1990612992, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1990621184, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 2186704896, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 2338093056, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 2489481216, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 2490005504, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 2494736384, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 2499467264, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 2504198144, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 2504206336, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 2504208384, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.4.attn_kv_b.weight", "offset": 2505176064, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 2506896384, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.4.attn_q.weight", "offset": 2510337024, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 2515497984, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 2515506176, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 2711589888, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 2862978048, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 3014366208, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 3014890496, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 3019621376, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 3024352256, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 3029083136, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 3029091328, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 3029093376, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.5.attn_kv_b.weight", "offset": 3030061056, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 3031781376, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.5.attn_q.weight", "offset": 3035222016, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 3040382976, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 3040391168, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 3236474880, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 3387863040, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 3539251200, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 3539775488, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 3544506368, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 3549237248, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 3553968128, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 3553976320, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 3553978368, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_kv_b.weight", "offset": 3554946048, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 3556666368, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.6.attn_q.weight", "offset": 3560107008, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 3565267968, "shape": [ 2048, 64 ], "size": 5160960, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 3565792256, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 3570523136, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 3575254016, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 3579984896, "shape": [ 512 ], "size": 4730880, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 3579986944, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.7.attn_kv_b.weight", "offset": 3580954624, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 3582674944, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.7.attn_q.weight", "offset": 3586115584, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "output_norm.weight", "offset": 3591276544, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 3591284736, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 3591292928, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 3787376640, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 3938764800, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 4090152960, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 4090677248, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 4095408128, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 4100139008, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 4104869888, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 4104878080, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 4104880128, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.10.attn_kv_b.weight", "offset": 4105847808, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 4107568128, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.10.attn_q.weight", "offset": 4111008768, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 4116169728, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 4116177920, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 4312261632, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 4463649792, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 4615037952, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 4615562240, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 4620293120, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 4625024000, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 4629754880, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 4629763072, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 4629765120, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.11.attn_kv_b.weight", "offset": 4630732800, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 4632453120, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.11.attn_q.weight", "offset": 4635893760, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 4641054720, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 4641062912, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 4837146624, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 4988534784, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 5139922944, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 5140447232, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 5145178112, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 5149908992, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 5154639872, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 5154648064, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 5154650112, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.12.attn_kv_b.weight", "offset": 5155617792, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 5157338112, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.12.attn_q.weight", "offset": 5160778752, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 5165939712, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 5165947904, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 5362031616, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 5513419776, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 5664807936, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 5665332224, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 5670063104, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 5674793984, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 5679524864, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 5679533056, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 5679535104, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.13.attn_kv_b.weight", "offset": 5680502784, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 5682223104, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.13.attn_q.weight", "offset": 5685663744, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 5690824704, "shape": [ 2048, 64 ], "size": 5160960, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 5691348992, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 5696079872, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 5700810752, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 5705541632, "shape": [ 512 ], "size": 4730880, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 5705543680, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.14.attn_kv_b.weight", "offset": 5706511360, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 5708231680, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.14.attn_q.weight", "offset": 5711672320, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 5716833280, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 5716841472, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 5912925184, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 6064313344, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 6215701504, "shape": [ 2048 ], "size": 151388160, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 6215709696, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 6215717888, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 6411801600, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 6563189760, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 6714577920, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 6715102208, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 6719833088, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 6724563968, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 6729294848, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 6729303040, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 6729305088, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.8.attn_kv_b.weight", "offset": 6730272768, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 6731993088, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.8.attn_q.weight", "offset": 6735433728, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 6740594688, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 6740602880, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 6936686592, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 7088074752, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 7239462912, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 7239987200, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 7244718080, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 7249448960, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.9.ffn_norm.weight", "offset": 7254179840, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 7254188032, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 7254190080, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.9.attn_kv_b.weight", "offset": 7255157760, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 7256878080, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.9.attn_q.weight", "offset": 7260318720, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 7265479680, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 7265487872, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 7461571584, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 7612959744, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 7764347904, "shape": [ 2048 ], "size": 151388160, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 7764356096, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 7764364288, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 7960448000, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 8111836160, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 8263224320, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 8263748608, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 8268479488, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 8273210368, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 8277941248, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 8277949440, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 8277951488, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.15.attn_kv_b.weight", "offset": 8278919168, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 8280639488, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.15.attn_q.weight", "offset": 8284080128, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 8289241088, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 8289249280, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 8485332992, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 8636721152, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 8788109312, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 8788633600, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 8793364480, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 8798095360, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 8802826240, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 8802834432, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 8802836480, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.16.attn_kv_b.weight", "offset": 8803804160, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 8805524480, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.16.attn_q.weight", "offset": 8808965120, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 8814126080, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 8814134272, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 9010217984, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 9161606144, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 9312994304, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 9313518592, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 9318249472, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 9322980352, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 9327711232, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 9327719424, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 9327721472, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.17.attn_kv_b.weight", "offset": 9328689152, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 9330409472, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.17.attn_q.weight", "offset": 9333850112, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 9339011072, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 9339019264, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 9535102976, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 9686491136, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 9837879296, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 9838403584, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 9843134464, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 9847865344, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 9852596224, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 9852604416, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 9852606464, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.18.attn_kv_b.weight", "offset": 9853574144, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 9855294464, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.18.attn_q.weight", "offset": 9858735104, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 9863896064, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 9863904256, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 10059987968, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 10211376128, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 10362764288, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 10363288576, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 10368019456, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 10372750336, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 10377481216, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 10377489408, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 10377491456, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.19.attn_kv_b.weight", "offset": 10378459136, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 10380179456, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.19.attn_q.weight", "offset": 10383620096, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 10388781056, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 10388789248, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 10584872960, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 10736261120, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 10887649280, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 10888173568, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 10892904448, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 10897635328, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 10902366208, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 10902374400, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 10902376448, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.20.attn_kv_b.weight", "offset": 10903344128, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 10905064448, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.20.attn_q.weight", "offset": 10908505088, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 10913666048, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 10913674240, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 11109757952, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 11261146112, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 11412534272, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 11413058560, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 11417789440, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 11422520320, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 11427251200, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 11427259392, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 11427261440, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.21.attn_kv_b.weight", "offset": 11428229120, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 11429949440, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.21.attn_q.weight", "offset": 11433390080, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 11438551040, "shape": [ 2048, 64 ], "size": 5160960, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 11439075328, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 11443806208, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 11448537088, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 11453267968, "shape": [ 512 ], "size": 4730880, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 11453270016, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.22.attn_kv_b.weight", "offset": 11454237696, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 11455958016, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.22.attn_q.weight", "offset": 11459398656, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 11464559616, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 11464567808, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 11660651520, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 11812039680, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 11963427840, "shape": [ 2048 ], "size": 151388160, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 11963436032, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 11963444224, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 12159527936, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 12310916096, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 12462304256, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 12462828544, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 12467559424, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 12472290304, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 12477021184, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 12477029376, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 12477031424, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.23.attn_kv_b.weight", "offset": 12477999104, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 12479719424, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.23.attn_q.weight", "offset": 12483160064, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 12488321024, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 12488329216, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 12684412928, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 12835801088, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 12987189248, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 12987713536, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 12992444416, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 12997175296, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 13001906176, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 13001914368, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 13001916416, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.24.attn_kv_b.weight", "offset": 13002884096, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 13004604416, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.24.attn_q.weight", "offset": 13008045056, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 13013206016, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 13013214208, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 13209297920, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 13360686080, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 13512074240, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 13512598528, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 13517329408, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 13522060288, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 13526791168, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 13526799360, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 13526801408, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.25.attn_kv_b.weight", "offset": 13527769088, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 13529489408, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.25.attn_q.weight", "offset": 13532930048, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 13538091008, "shape": [ 2048 ], "size": 5160960, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 13538099200, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 13734182912, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 14 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 13885571072, "shape": [ 2048, 1408, 64 ], "size": 151388160, "type": 14 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 14036959232, "shape": [ 2048, 64 ], "size": 151388160, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 14037483520, "shape": [ 2816, 2048 ], "size": 524288, "type": 14 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 14042214400, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 14046945280, "shape": [ 2048, 2816 ], "size": 4730880, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 14051676160, "shape": [ 2048 ], "size": 4730880, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 14051684352, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 14051686400, "shape": [ 2048, 576 ], "size": 2048, "type": 14 }, { "name": "blk.26.attn_kv_b.weight", "offset": 14052654080, "shape": [ 512, 4096 ], "size": 967680, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 14054374400, "shape": [ 2048, 2048 ], "size": 1720320, "type": 14 }, { "name": "blk.26.attn_q.weight", "offset": 14057815040, "shape": [ 2048, 3072 ], "size": 3440640, "type": 14 } ], "version": 3 }