deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 9, "general.name": "DeepSeek-Coder-V2-Lite-Base", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 7 }, { "name": "blk.0.attn_norm.weight", "offset": 329318400, "shape": [ 2048 ], "size": 157286400, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 329326592, "shape": [ 10944, 2048 ], "size": 8192, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 346136576, "shape": [ 2048, 10944 ], "size": 16809984, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 362946560, "shape": [ 2048, 10944 ], "size": 16809984, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 379756544, "shape": [ 2048 ], "size": 16809984, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 379764736, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 379766784, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.0.attn_kv_b.weight", "offset": 380651520, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 382224384, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.0.attn_q.weight", "offset": 385370112, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 390088704, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 390096896, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 528508928, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 666920960, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 805332992, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 805857280, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 810182656, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 814508032, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 818833408, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 818841600, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 818843648, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.1.attn_kv_b.weight", "offset": 819728384, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 821301248, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.1.attn_q.weight", "offset": 824446976, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 829165568, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 829173760, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 967585792, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 1105997824, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1244409856, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 1244934144, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 1249259520, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 1253584896, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 1257910272, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 1257918464, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 1257920512, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.2.attn_kv_b.weight", "offset": 1258805248, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 1260378112, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.2.attn_q.weight", "offset": 1263523840, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 1268242432, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 1268250624, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1406662656, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1545074688, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1683486720, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1684011008, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1688336384, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1692661760, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 1696987136, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1696995328, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1696997376, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1697882112, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 1699454976, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.3.attn_q.weight", "offset": 1702600704, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 1707319296, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1707327488, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1845739520, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1984151552, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 2122563584, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 2123087872, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 2127413248, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 2131738624, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 2136064000, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 2136072192, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 2136074240, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.4.attn_kv_b.weight", "offset": 2136958976, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 2138531840, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.4.attn_q.weight", "offset": 2141677568, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 2146396160, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 2146404352, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 2284816384, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 2423228416, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 2561640448, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 2562164736, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 2566490112, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 2570815488, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 2575140864, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 2575149056, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 2575151104, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.5.attn_kv_b.weight", "offset": 2576035840, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 2577608704, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.5.attn_q.weight", "offset": 2580754432, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 2585473024, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 2585481216, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 2723893248, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 2862305280, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "wn_shexp.weight\u0002\u0000\u0000\u0000\u0000\u000b\u0000\u0000\u0000\u0000", "offset": 3000717312, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 3001241600, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 3005566976, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 3009892352, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 3014217728, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 3014225920, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 3014227968, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.6.attn_kv_b.weight", "offset": 3015112704, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 3016685568, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.6.attn_q.weight", "offset": 3019831296, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 3024549888, "shape": [ 2048, 64 ], "size": 4718592, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 3025074176, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 3029399552, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 3033724928, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 3038050304, "shape": [ 512 ], "size": 4325376, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 3038052352, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.7.attn_kv_b.weight", "offset": 3038937088, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 3040509952, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.7.attn_q.weight", "offset": 3043655680, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "output_norm.weight", "offset": 3048374272, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 3048382464, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 3048390656, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 3186802688, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 3325214720, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 3463626752, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 3464151040, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 3468476416, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 3472801792, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 3477127168, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 3477135360, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 3477137408, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.10.attn_kv_b.weight", "offset": 3478022144, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 3479595008, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.10.attn_q.weight", "offset": 3482740736, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 3487459328, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 3487467520, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 3625879552, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 3764291584, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 3902703616, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 3903227904, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 3907553280, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 3911878656, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 3916204032, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 3916212224, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 3916214272, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.11.attn_kv_b.weight", "offset": 3917099008, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 3918671872, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.11.attn_q.weight", "offset": 3921817600, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 3926536192, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 3926544384, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 4064956416, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 4203368448, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 4341780480, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 4342304768, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 4346630144, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 4350955520, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 4355280896, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 4355289088, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 4355291136, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.12.attn_kv_b.weight", "offset": 4356175872, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 4357748736, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.12.attn_q.weight", "offset": 4360894464, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 4365613056, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 4365621248, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 4504033280, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 4642445312, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 4780857344, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 4781381632, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 4785707008, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 4790032384, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 4794357760, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 4794365952, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 4794368000, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.13.attn_kv_b.weight", "offset": 4795252736, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 4796825600, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.13.attn_q.weight", "offset": 4799971328, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 4804689920, "shape": [ 2048, 64 ], "size": 4718592, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 4805214208, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 4809539584, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 4813864960, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 4818190336, "shape": [ 512 ], "size": 4325376, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 4818192384, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.14.attn_kv_b.weight", "offset": 4819077120, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 4820649984, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.14.attn_q.weight", "offset": 4823795712, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 4828514304, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 4828522496, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 4966934528, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 5105346560, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 5243758592, "shape": [ 2048 ], "size": 138412032, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 5243766784, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 5243774976, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 5382187008, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 5520599040, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 5659011072, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 5659535360, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 5663860736, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 5668186112, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 5672511488, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 5672519680, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 5672521728, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.8.attn_kv_b.weight", "offset": 5673406464, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 5674979328, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.8.attn_q.weight", "offset": 5678125056, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 5682843648, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 5682851840, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 5821263872, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 5959675904, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 6098087936, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 6098612224, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 6102937600, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 6107262976, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 6111588352, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 6111596544, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 6111598592, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.9.attn_kv_b.weight", "offset": 6112483328, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 6114056192, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.9.attn_q.weight", "offset": 6117201920, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 6121920512, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 6121928704, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 6260340736, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 6398752768, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 6537164800, "shape": [ 2048 ], "size": 138412032, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 6537172992, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 6537181184, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 6675593216, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "\u0000\u0000\u0000\u0000\b\u0000\u0000\u0000\u0000\u0000\u0000\ufffd\u0005\u0000\u0000\u0000\u0000\u0000\u0000@\u0000\u0000\u0000\u0000\u0000", "offset": 6814005248, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 6952417280, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 6952941568, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 6957266944, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 6961592320, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 6965917696, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 6965925888, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 6965927936, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.15.attn_kv_b.weight", "offset": 6966812672, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 6968385536, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.15.attn_q.weight", "offset": 6971531264, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 6976249856, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 6976258048, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 7114670080, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 7253082112, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 7391494144, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 7392018432, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 7396343808, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 7400669184, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 7404994560, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 7405002752, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 7405004800, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.16.attn_kv_b.weight", "offset": 7405889536, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 7407462400, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.16.attn_q.weight", "offset": 7410608128, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 7415326720, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 7415334912, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 7553746944, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 7692158976, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 7830571008, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 7831095296, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 7835420672, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 7839746048, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 7844071424, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 7844079616, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 7844081664, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.17.attn_kv_b.weight", "offset": 7844966400, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 7846539264, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.17.attn_q.weight", "offset": 7849684992, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 7854403584, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 7854411776, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 7992823808, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 8131235840, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 8269647872, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 8270172160, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 8274497536, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 8278822912, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 8283148288, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 8283156480, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 8283158528, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.18.attn_kv_b.weight", "offset": 8284043264, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 8285616128, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.18.attn_q.weight", "offset": 8288761856, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 8293480448, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 8293488640, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 8431900672, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 8570312704, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 8708724736, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 8709249024, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 8713574400, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 8717899776, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 8722225152, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 8722233344, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 8722235392, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.19.attn_kv_b.weight", "offset": 8723120128, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 8724692992, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.19.attn_q.weight", "offset": 8727838720, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 8732557312, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 8732565504, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 8870977536, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 9009389568, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 9147801600, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 9148325888, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 9152651264, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 9156976640, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 9161302016, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 9161310208, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 9161312256, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.20.attn_kv_b.weight", "offset": 9162196992, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 9163769856, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.20.attn_q.weight", "offset": 9166915584, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 9171634176, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 9171642368, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 9310054400, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 9448466432, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 9586878464, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 9587402752, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 9591728128, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 9596053504, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 9600378880, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 9600387072, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 9600389120, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.21.attn_kv_b.weight", "offset": 9601273856, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 9602846720, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.21.attn_q.weight", "offset": 9605992448, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 9610711040, "shape": [ 2048, 64 ], "size": 4718592, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 9611235328, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 9615560704, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 9619886080, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 9624211456, "shape": [ 512 ], "size": 4325376, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 9624213504, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.22.attn_kv_b.weight", "offset": 9625098240, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 9626671104, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.22.attn_q.weight", "offset": 9629816832, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 9634535424, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 9634543616, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 9772955648, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 9911367680, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 10049779712, "shape": [ 2048 ], "size": 138412032, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 10049787904, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 10049796096, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 10188208128, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 10326620160, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 10465032192, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 10465556480, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 10469881856, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 10474207232, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 10478532608, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 10478540800, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 10478542848, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.23.attn_kv_b.weight", "offset": 10479427584, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 10481000448, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.23.attn_q.weight", "offset": 10484146176, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 10488864768, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 10488872960, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 10627284992, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 10765697024, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 10904109056, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 10904633344, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 10908958720, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 10913284096, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 10917609472, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 10917617664, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 10917619712, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.24.attn_kv_b.weight", "offset": 10918504448, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 10920077312, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.24.attn_q.weight", "offset": 10923223040, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 10927941632, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 10927949824, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 11066361856, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 11204773888, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 11343185920, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 11343710208, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 11348035584, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 11352360960, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 11356686336, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 11356694528, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 11356696576, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.25.attn_kv_b.weight", "offset": 11357581312, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 11359154176, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.25.attn_q.weight", "offset": 11362299904, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 11367018496, "shape": [ 2048 ], "size": 4718592, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 11367026688, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 7 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 11505438720, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 11643850752, "shape": [ 2048, 1408, 64 ], "size": 138412032, "type": 7 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 11782262784, "shape": [ 2048, 64 ], "size": 138412032, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 11782787072, "shape": [ 2816, 2048 ], "size": 524288, "type": 7 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 11787112448, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 11791437824, "shape": [ 2048, 2816 ], "size": 4325376, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 11795763200, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 11795771392, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 11795773440, "shape": [ 2048, 576 ], "size": 2048, "type": 7 }, { "name": "blk.26.attn_kv_b.weight", "offset": 11796658176, "shape": [ 512, 4096 ], "size": 884736, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 11798231040, "shape": [ 2048, 2048 ], "size": 1572864, "type": 7 }, { "name": "blk.26.attn_q.weight", "offset": 11801376768, "shape": [ 2048, 3072 ], "size": 3145728, "type": 7 } ], "version": 3 }