deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 3, "general.name": "DeepSeek-Coder-V2-Lite-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 3 }, { "name": "blk.0.attn_norm.weight", "offset": 303104000, "shape": [ 2048 ], "size": 131072000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 303112192, "shape": [ 10944, 2048 ], "size": 8192, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 317120512, "shape": [ 2048, 10944 ], "size": 14008320, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 331128832, "shape": [ 2048, 10944 ], "size": 14008320, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 345137152, "shape": [ 2048 ], "size": 14008320, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 345145344, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 345147392, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.0.attn_kv_b.weight", "offset": 345884672, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 347195392, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.0.attn_q.weight", "offset": 349816832, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 353748992, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 353757184, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 469100544, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 584443904, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 699787264, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 700311552, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 703916032, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 707520512, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 711124992, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 711133184, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 711135232, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.1.attn_kv_b.weight", "offset": 711872512, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 713183232, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.1.attn_q.weight", "offset": 715804672, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 719736832, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 719745024, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 835088384, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 950431744, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1065775104, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 1066299392, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 1069903872, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 1073508352, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 1077112832, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 1077121024, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 1077123072, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.2.attn_kv_b.weight", "offset": 1077860352, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 1079171072, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.2.attn_q.weight", "offset": 1081792512, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 1085724672, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 1085732864, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1201076224, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1316419584, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1431762944, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1432287232, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1435891712, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1439496192, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 1443100672, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1443108864, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1443110912, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1443848192, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 1445158912, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.3.attn_q.weight", "offset": 1447780352, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 1451712512, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1451720704, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1567064064, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1682407424, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 1797750784, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 1798275072, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 1801879552, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 1805484032, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 1809088512, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 1809096704, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 1809098752, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.4.attn_kv_b.weight", "offset": 1809836032, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 1811146752, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.4.attn_q.weight", "offset": 1813768192, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 1817700352, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 1817708544, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 1933051904, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 2048395264, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 2163738624, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 2164262912, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 2167867392, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 2171471872, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 2175076352, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 2175084544, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 2175086592, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.5.attn_kv_b.weight", "offset": 2175823872, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 2177134592, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.5.attn_q.weight", "offset": 2179756032, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 2183688192, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 2183696384, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 2299039744, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 2414383104, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "n_down_shexp.weight\u0002\u0000\u0000\u0000\u0000\u000b", "offset": 2529726464, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 2530250752, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 2533855232, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 2537459712, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 2541064192, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 2541072384, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 2541074432, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.6.attn_kv_b.weight", "offset": 2541811712, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 2543122432, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.6.attn_q.weight", "offset": 2545743872, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 2549676032, "shape": [ 2048, 64 ], "size": 3932160, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 2550200320, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 2553804800, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 2557409280, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 2561013760, "shape": [ 512 ], "size": 3604480, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 2561015808, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.7.attn_kv_b.weight", "offset": 2561753088, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 2563063808, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.7.attn_q.weight", "offset": 2565685248, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "output_norm.weight", "offset": 2569617408, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 2569625600, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 2569633792, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 2684977152, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 2800320512, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 2915663872, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 2916188160, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 2919792640, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 2923397120, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 2927001600, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 2927009792, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 2927011840, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.10.attn_kv_b.weight", "offset": 2927749120, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 2929059840, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.10.attn_q.weight", "offset": 2931681280, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 2935613440, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 2935621632, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 3050964992, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 3166308352, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 3281651712, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 3282176000, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 3285780480, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 3289384960, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 3292989440, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 3292997632, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 3292999680, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.11.attn_kv_b.weight", "offset": 3293736960, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 3295047680, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.11.attn_q.weight", "offset": 3297669120, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 3301601280, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 3301609472, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 3416952832, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 3532296192, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 3647639552, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 3648163840, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 3651768320, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 3655372800, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 3658977280, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 3658985472, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 3658987520, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.12.attn_kv_b.weight", "offset": 3659724800, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 3661035520, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.12.attn_q.weight", "offset": 3663656960, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 3667589120, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 3667597312, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 3782940672, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 3898284032, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 4013627392, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 4014151680, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 4017756160, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 4021360640, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 4024965120, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 4024973312, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 4024975360, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.13.attn_kv_b.weight", "offset": 4025712640, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 4027023360, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.13.attn_q.weight", "offset": 4029644800, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 4033576960, "shape": [ 2048, 64 ], "size": 3932160, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 4034101248, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 4037705728, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 4041310208, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 4044914688, "shape": [ 512 ], "size": 3604480, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 4044916736, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.14.attn_kv_b.weight", "offset": 4045654016, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 4046964736, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.14.attn_q.weight", "offset": 4049586176, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.7.attn_norm.weight", "offset": 4053518336, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 4053526528, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 4168869888, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 4284213248, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 4399556608, "shape": [ 2048 ], "size": 115343360, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 4399564800, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 4399572992, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 4514916352, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 4630259712, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 4745603072, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 4746127360, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 4749731840, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 4753336320, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 4756940800, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 4756948992, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 4756951040, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.8.attn_kv_b.weight", "offset": 4757688320, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 4758999040, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.8.attn_q.weight", "offset": 4761620480, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 4765552640, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 4765560832, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 4880904192, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 4996247552, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 5111590912, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 5112115200, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 5115719680, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 5119324160, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 5122928640, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 5122936832, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 5122938880, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.9.attn_kv_b.weight", "offset": 5123676160, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 5124986880, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.9.attn_q.weight", "offset": 5127608320, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 5131540480, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 5131548672, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 5246892032, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 5362235392, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 5477578752, "shape": [ 2048 ], "size": 115343360, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 5477586944, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 5477595136, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 5592938496, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "\b\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0005\u0000\u0000\u0000\u0000\u0000\u0000@\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003\u0000", "offset": 5708281856, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 5823625216, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 5824149504, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 5827753984, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 5831358464, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 5834962944, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 5834971136, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 5834973184, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.15.attn_kv_b.weight", "offset": 5835710464, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 5837021184, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.15.attn_q.weight", "offset": 5839642624, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 5843574784, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 5843582976, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 5958926336, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 6074269696, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 6189613056, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 6190137344, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 6193741824, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 6197346304, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 6200950784, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 6200958976, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 6200961024, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.16.attn_kv_b.weight", "offset": 6201698304, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 6203009024, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.16.attn_q.weight", "offset": 6205630464, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 6209562624, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 6209570816, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 6324914176, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 6440257536, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 6555600896, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 6556125184, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 6559729664, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 6563334144, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 6566938624, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 6566946816, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 6566948864, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.17.attn_kv_b.weight", "offset": 6567686144, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 6568996864, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.17.attn_q.weight", "offset": 6571618304, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 6575550464, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 6575558656, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 6690902016, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 6806245376, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 6921588736, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 6922113024, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 6925717504, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 6929321984, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 6932926464, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 6932934656, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 6932936704, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.18.attn_kv_b.weight", "offset": 6933673984, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 6934984704, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.18.attn_q.weight", "offset": 6937606144, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 6941538304, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 6941546496, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 7056889856, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 7172233216, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 7287576576, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 7288100864, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 7291705344, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 7295309824, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 7298914304, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 7298922496, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 7298924544, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.19.attn_kv_b.weight", "offset": 7299661824, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 7300972544, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.19.attn_q.weight", "offset": 7303593984, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 7307526144, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 7307534336, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 7422877696, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 7538221056, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 7653564416, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 7654088704, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 7657693184, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 7661297664, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 7664902144, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 7664910336, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 7664912384, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.20.attn_kv_b.weight", "offset": 7665649664, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 7666960384, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.20.attn_q.weight", "offset": 7669581824, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 7673513984, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 7673522176, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 7788865536, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 7904208896, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 8019552256, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 8020076544, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 8023681024, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 8027285504, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 8030889984, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 8030898176, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 8030900224, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.21.attn_kv_b.weight", "offset": 8031637504, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 8032948224, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.21.attn_q.weight", "offset": 8035569664, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 8039501824, "shape": [ 2048, 64 ], "size": 3932160, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 8040026112, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 8043630592, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 8047235072, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 8050839552, "shape": [ 512 ], "size": 3604480, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 8050841600, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.22.attn_kv_b.weight", "offset": 8051578880, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 8052889600, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.22.attn_q.weight", "offset": 8055511040, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.22.attn_norm.weight", "offset": 8059443200, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 8059451392, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 8174794752, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 8290138112, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 8405481472, "shape": [ 2048 ], "size": 115343360, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 8405489664, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 8405497856, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 8520841216, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 8636184576, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 8751527936, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 8752052224, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 8755656704, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 8759261184, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 8762865664, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 8762873856, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 8762875904, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.23.attn_kv_b.weight", "offset": 8763613184, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 8764923904, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.23.attn_q.weight", "offset": 8767545344, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 8771477504, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 8771485696, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 8886829056, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 9002172416, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 9117515776, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 9118040064, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 9121644544, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 9125249024, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 9128853504, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 9128861696, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 9128863744, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.24.attn_kv_b.weight", "offset": 9129601024, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 9130911744, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.24.attn_q.weight", "offset": 9133533184, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 9137465344, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 9137473536, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 9252816896, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 9368160256, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 9483503616, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 9484027904, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 9487632384, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 9491236864, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 9494841344, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 9494849536, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 9494851584, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.25.attn_kv_b.weight", "offset": 9495588864, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 9496899584, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.25.attn_q.weight", "offset": 9499521024, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 9503453184, "shape": [ 2048 ], "size": 3932160, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 9503461376, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 3 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 9618804736, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 9734148096, "shape": [ 2048, 1408, 64 ], "size": 115343360, "type": 3 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 9849491456, "shape": [ 2048, 64 ], "size": 115343360, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 9850015744, "shape": [ 2816, 2048 ], "size": 524288, "type": 3 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 9853620224, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 9857224704, "shape": [ 2048, 2816 ], "size": 3604480, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 9860829184, "shape": [ 2048 ], "size": 3604480, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 9860837376, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 9860839424, "shape": [ 2048, 576 ], "size": 2048, "type": 3 }, { "name": "blk.26.attn_kv_b.weight", "offset": 9861576704, "shape": [ 512, 4096 ], "size": 737280, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 9862887424, "shape": [ 2048, 2048 ], "size": 1310720, "type": 3 }, { "name": "blk.26.attn_q.weight", "offset": 9865508864, "shape": [ 2048, 3072 ], "size": 2621440, "type": 3 } ], "version": 3 }