deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 7, "general.name": "DeepSeek-Coder-V2-Lite-Base", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 222822400, "shape": [ 2048, 102400 ], "size": 222822400, "type": 8 }, { "name": "blk.0.attn_norm.weight", "offset": 445644800, "shape": [ 2048 ], "size": 222822400, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 445652992, "shape": [ 10944, 2048 ], "size": 8192, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 469467136, "shape": [ 2048, 10944 ], "size": 23814144, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 493281280, "shape": [ 2048, 10944 ], "size": 23814144, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 517095424, "shape": [ 2048 ], "size": 23814144, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 517103616, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 517105664, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.0.attn_kv_b.weight", "offset": 518359040, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 520587264, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.0.attn_q.weight", "offset": 525043712, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 531728384, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 531736576, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 727820288, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 923904000, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1119987712, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 1120512000, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 1126639616, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 1132767232, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 1138894848, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 1138903040, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 1138905088, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.1.attn_kv_b.weight", "offset": 1140158464, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1142386688, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.1.attn_q.weight", "offset": 1146843136, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 1153527808, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 1153536000, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 1349619712, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 1545703424, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1741787136, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 1742311424, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 1748439040, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 1754566656, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 1760694272, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 1760702464, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 1760704512, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.2.attn_kv_b.weight", "offset": 1761957888, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1764186112, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.2.attn_q.weight", "offset": 1768642560, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 1775327232, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 1775335424, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1971419136, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 2167502848, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 2363586560, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 2364110848, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 2370238464, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 2376366080, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 2382493696, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 2382501888, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 2382503936, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.3.attn_kv_b.weight", "offset": 2383757312, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 2385985536, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.3.attn_q.weight", "offset": 2390441984, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 2397126656, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 2397134848, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 2593218560, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 2789302272, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 2985385984, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 2985910272, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 2992037888, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 2998165504, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 3004293120, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 3004301312, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 3004303360, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.4.attn_kv_b.weight", "offset": 3005556736, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 3007784960, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.4.attn_q.weight", "offset": 3012241408, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 3018926080, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 3018934272, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 3215017984, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 3411101696, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 3607185408, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 3607709696, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 3613837312, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 3619964928, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 3626092544, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 3626100736, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 3626102784, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.5.attn_kv_b.weight", "offset": 3627356160, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 3629584384, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.5.attn_q.weight", "offset": 3634040832, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 3640725504, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 3640733696, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 3836817408, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 4032901120, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "\u0005\u0000\u0000\u0000\u0000\u0000\u0000@\u0000\u0000\u0000\u0000\u0000\u0000\u0000\b\u0000\u0000\u0000\u0000\ufffd\ufffdz\u0001\u0000", "offset": 4228984832, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 4229509120, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 4235636736, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 4241764352, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 4247891968, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 4247900160, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 4247902208, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.6.attn_kv_b.weight", "offset": 4249155584, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 4251383808, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.6.attn_q.weight", "offset": 4255840256, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 4262524928, "shape": [ 2048, 64 ], "size": 6684672, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 4263049216, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 4269176832, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 4275304448, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 4281432064, "shape": [ 512 ], "size": 6127616, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 4281434112, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.7.attn_kv_b.weight", "offset": 4282687488, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 4284915712, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.7.attn_q.weight", "offset": 4289372160, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "output_norm.weight", "offset": 4296056832, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 4296065024, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 4296073216, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 4492156928, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 4688240640, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 4884324352, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 4884848640, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 4890976256, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 4897103872, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 4903231488, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 4903239680, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 4903241728, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.10.attn_kv_b.weight", "offset": 4904495104, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 4906723328, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.10.attn_q.weight", "offset": 4911179776, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 4917864448, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 4917872640, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 5113956352, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 5310040064, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 5506123776, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 5506648064, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 5512775680, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 5518903296, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 5525030912, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 5525039104, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 5525041152, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.11.attn_kv_b.weight", "offset": 5526294528, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 5528522752, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.11.attn_q.weight", "offset": 5532979200, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 5539663872, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 5539672064, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 5735755776, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 5931839488, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 6127923200, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 6128447488, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 6134575104, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 6140702720, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 6146830336, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 6146838528, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 6146840576, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.12.attn_kv_b.weight", "offset": 6148093952, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 6150322176, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.12.attn_q.weight", "offset": 6154778624, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 6161463296, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 6161471488, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 6357555200, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 6553638912, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 6749722624, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 6750246912, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 6756374528, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 6762502144, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 6768629760, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 6768637952, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 6768640000, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.13.attn_kv_b.weight", "offset": 6769893376, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 6772121600, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.13.attn_q.weight", "offset": 6776578048, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 6783262720, "shape": [ 2048, 64 ], "size": 6684672, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 6783787008, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 6789914624, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 6796042240, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 6802169856, "shape": [ 512 ], "size": 6127616, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 6802171904, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.14.attn_kv_b.weight", "offset": 6803425280, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 6805653504, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.14.attn_q.weight", "offset": 6810109952, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 6816794624, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 6816802816, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 7012886528, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 7208970240, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 7405053952, "shape": [ 2048 ], "size": 196083712, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 7405062144, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 7405070336, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 7601154048, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 7797237760, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 7993321472, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 7993845760, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 7999973376, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 8006100992, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 8012228608, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 8012236800, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 8012238848, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.8.attn_kv_b.weight", "offset": 8013492224, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 8015720448, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.8.attn_q.weight", "offset": 8020176896, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 8026861568, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 8026869760, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 8222953472, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 8419037184, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 8615120896, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 8615645184, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 8621772800, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 8627900416, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 8634028032, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 8634036224, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 8634038272, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.9.attn_kv_b.weight", "offset": 8635291648, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 8637519872, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.9.attn_q.weight", "offset": 8641976320, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 8648660992, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 8648669184, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 8844752896, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 9040836608, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 9236920320, "shape": [ 2048 ], "size": 196083712, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 9236928512, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 9236936704, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 9433020416, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "\u0000\u0000\u0000\u0000\b\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0005\u0000\u0000\u0000\u0000\u0000\u0000@\u0000\u0000\u0000\u0000\u0000", "offset": 9629104128, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 9825187840, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 9825712128, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 9831839744, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 9837967360, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 9844094976, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 9844103168, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 9844105216, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.15.attn_kv_b.weight", "offset": 9845358592, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 9847586816, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.15.attn_q.weight", "offset": 9852043264, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 9858727936, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 9858736128, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 10054819840, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 10250903552, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 10446987264, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 10447511552, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 10453639168, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 10459766784, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 10465894400, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 10465902592, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 10465904640, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.16.attn_kv_b.weight", "offset": 10467158016, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 10469386240, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.16.attn_q.weight", "offset": 10473842688, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 10480527360, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 10480535552, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 10676619264, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 10872702976, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 11068786688, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 11069310976, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 11075438592, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 11081566208, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 11087693824, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 11087702016, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 11087704064, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.17.attn_kv_b.weight", "offset": 11088957440, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 11091185664, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.17.attn_q.weight", "offset": 11095642112, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 11102326784, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 11102334976, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 11298418688, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 11494502400, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 11690586112, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 11691110400, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 11697238016, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 11703365632, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 11709493248, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 11709501440, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 11709503488, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.18.attn_kv_b.weight", "offset": 11710756864, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 11712985088, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.18.attn_q.weight", "offset": 11717441536, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 11724126208, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 11724134400, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 11920218112, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 12116301824, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 12312385536, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 12312909824, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 12319037440, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 12325165056, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 12331292672, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 12331300864, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 12331302912, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.19.attn_kv_b.weight", "offset": 12332556288, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 12334784512, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.19.attn_q.weight", "offset": 12339240960, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 12345925632, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 12345933824, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 12542017536, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 12738101248, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 12934184960, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 12934709248, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 12940836864, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 12946964480, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 12953092096, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 12953100288, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 12953102336, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.20.attn_kv_b.weight", "offset": 12954355712, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 12956583936, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.20.attn_q.weight", "offset": 12961040384, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 12967725056, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 12967733248, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 13163816960, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 13359900672, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 13555984384, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 13556508672, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 13562636288, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 13568763904, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 13574891520, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 13574899712, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 13574901760, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.21.attn_kv_b.weight", "offset": 13576155136, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 13578383360, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.21.attn_q.weight", "offset": 13582839808, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 13589524480, "shape": [ 2048, 64 ], "size": 6684672, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 13590048768, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 13596176384, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 13602304000, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 13608431616, "shape": [ 512 ], "size": 6127616, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 13608433664, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.22.attn_kv_b.weight", "offset": 13609687040, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 13611915264, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.22.attn_q.weight", "offset": 13616371712, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 13623056384, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 13623064576, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 13819148288, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 14015232000, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 14211315712, "shape": [ 2048 ], "size": 196083712, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 14211323904, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 14211332096, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 14407415808, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 14603499520, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 14799583232, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 14800107520, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 14806235136, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 14812362752, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 14818490368, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 14818498560, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 14818500608, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.23.attn_kv_b.weight", "offset": 14819753984, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 14821982208, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.23.attn_q.weight", "offset": 14826438656, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 14833123328, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 14833131520, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 15029215232, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 15225298944, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 15421382656, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 15421906944, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 15428034560, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 15434162176, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 15440289792, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 15440297984, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 15440300032, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.24.attn_kv_b.weight", "offset": 15441553408, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 15443781632, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.24.attn_q.weight", "offset": 15448238080, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 15454922752, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 15454930944, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 15651014656, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 15847098368, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 16043182080, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 16043706368, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 16049833984, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 16055961600, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 16062089216, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 16062097408, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 16062099456, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.25.attn_kv_b.weight", "offset": 16063352832, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 16065581056, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.25.attn_q.weight", "offset": 16070037504, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 16076722176, "shape": [ 2048 ], "size": 6684672, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 16076730368, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 8 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 16272814080, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 16468897792, "shape": [ 2048, 1408, 64 ], "size": 196083712, "type": 8 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 16664981504, "shape": [ 2048, 64 ], "size": 196083712, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 16665505792, "shape": [ 2816, 2048 ], "size": 524288, "type": 8 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 16671633408, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 16677761024, "shape": [ 2048, 2816 ], "size": 6127616, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 16683888640, "shape": [ 2048 ], "size": 6127616, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 16683896832, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 16683898880, "shape": [ 2048, 576 ], "size": 2048, "type": 8 }, { "name": "blk.26.attn_kv_b.weight", "offset": 16685152256, "shape": [ 512, 4096 ], "size": 1253376, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 16687380480, "shape": [ 2048, 2048 ], "size": 2228224, "type": 8 }, { "name": "blk.26.attn_q.weight", "offset": 16691836928, "shape": [ 2048, 3072 ], "size": 4456448, "type": 8 } ], "version": 3 }