deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 128, "deepseek2.attention.head_count_kv": 128, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.q_lora_rank": 1536, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 60, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 5120, "deepseek2.expert_count": 160, "deepseek2.expert_feed_forward_length": 1536, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 16, "deepseek2.feed_forward_length": 12288, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.1, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 9, "general.name": "DeepSeek-Coder-V2-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 235741434880, "tensors": [ { "name": "token_embd.weight", "offset": 430080000, "shape": [ 5120, 102400 ], "size": 430080000, "type": 7 }, { "name": "blk.0.attn_norm.weight", "offset": 823296000, "shape": [ 5120 ], "size": 393216000, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 823316480, "shape": [ 12288, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 870502400, "shape": [ 5120, 12288 ], "size": 47185920, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 917688320, "shape": [ 5120, 12288 ], "size": 47185920, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 964874240, "shape": [ 5120 ], "size": 47185920, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 964894720, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 964896768, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.0.attn_kv_b.weight", "offset": 967108608, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 979691520, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.0.attn_q_a_norm.weight", "offset": 1042606080, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.0.attn_q_a.weight", "offset": 1042612224, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.0.attn_q_b.weight", "offset": 1048510464, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 1076822016, "shape": [ 5120, 160 ], "size": 28311552, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 1080098816, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 1091895296, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 1103691776, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 1115488256, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 1115490304, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.1.attn_kv_b.weight", "offset": 1117702144, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 1130285056, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\u0000\u0000\u0007\u0000\u0000\u0000\u0000\ufffds\ufffd\u0003\u0000\u0000\u0000", "offset": 1193199616, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.1.attn_q_a.weight", "offset": 1193205760, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.1.attn_q_b.weight", "offset": 1199104000, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "output_norm.weight", "offset": 1227415552, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.1.attn_norm.weight", "offset": 1227436032, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 1227456512, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 2171174912, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 3114893312, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 4058611712, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 4058632192, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 4061908992, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 4073705472, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 4085501952, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 4097298432, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 4097300480, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.2.attn_kv_b.weight", "offset": 4099512320, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 4112095232, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.2.attn_q_a_norm.weight", "offset": 4175009792, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.2.attn_q_a.weight", "offset": 4175015936, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.2.attn_q_b.weight", "offset": 4180914176, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 4209225728, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 4209246208, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 5152964608, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 6096683008, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 7040401408, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 7040421888, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 7043698688, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 7055495168, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 7067291648, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 7079088128, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 7079090176, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.3.attn_kv_b.weight", "offset": 7081302016, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 7093884928, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.3.attn_q_a_norm.weight", "offset": 7156799488, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.3.attn_q_a.weight", "offset": 7156805632, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.3.attn_q_b.weight", "offset": 7162703872, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 7191015424, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 7191035904, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 8134754304, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 9078472704, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 10022191104, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 10022211584, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 10025488384, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 10037284864, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 10049081344, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 10060877824, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 10060879872, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.4.attn_kv_b.weight", "offset": 10063091712, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 10075674624, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.4.attn_q_a_norm.weight", "offset": 10138589184, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.4.attn_q_a.weight", "offset": 10138595328, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.4.attn_q_b.weight", "offset": 10144493568, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 10172805120, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 10172825600, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 11116544000, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 12060262400, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 13003980800, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.5.attn_norm.weight", "offset": 13004001280, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 13004021760, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 13947740160, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 14891458560, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 15835176960, "shape": [ 5120, 160 ], "size": 943718400, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 15838453760, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 15850250240, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 15862046720, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 15873843200, "shape": [ 5120 ], "size": 11796480, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 15873863680, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 15873865728, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.5.attn_kv_b.weight", "offset": 15876077568, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 15888660480, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.5.attn_q_a_norm.weight", "offset": 15951575040, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.5.attn_q_a.weight", "offset": 15951581184, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.5.attn_q_b.weight", "offset": 15957479424, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 15985790976, "shape": [ 5120, 160 ], "size": 28311552, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 15989067776, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 16000864256, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 16012660736, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 16024457216, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 16024459264, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.6.attn_kv_b.weight", "offset": 16026671104, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 16039254016, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.6.attn_q_a_norm.weight", "offset": 16102168576, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.6.attn_q_a.weight", "offset": 16102174720, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.6.attn_q_b.weight", "offset": 16108072960, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 16136384512, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 16136404992, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 17080123392, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 18023841792, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 18967560192, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 18967580672, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 18970857472, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 18982653952, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 18994450432, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 19006246912, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 19006248960, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.7.attn_kv_b.weight", "offset": 19008460800, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 19021043712, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.7.attn_q_a_norm.weight", "offset": 19083958272, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.7.attn_q_a.weight", "offset": 19083964416, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.7.attn_q_b.weight", "offset": 19089862656, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 19118174208, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 19118194688, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 20061913088, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 21005631488, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 21949349888, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 21949370368, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 21952647168, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 21964443648, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 21976240128, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 21988036608, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 21988038656, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.8.attn_kv_b.weight", "offset": 21990250496, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 22002833408, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.8.attn_q_a_norm.weight", "offset": 22065747968, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.8.attn_q_a.weight", "offset": 22065754112, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.8.attn_q_b.weight", "offset": 22071652352, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 22099963904, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 22099984384, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 23043702784, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 23987421184, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 24931139584, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 24931160064, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 24934436864, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 24946233344, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 24958029824, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 24969826304, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 24969828352, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.9.attn_kv_b.weight", "offset": 24972040192, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 24984623104, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.9.attn_q_a_norm.weight", "offset": 25047537664, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.9.attn_q_a.weight", "offset": 25047543808, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.9.attn_q_b.weight", "offset": 25053442048, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 25081753600, "shape": [ 5120, 160 ], "size": 28311552, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 25085030400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 25096826880, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 25108623360, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 25120419840, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 25120421888, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.10.attn_kv_b.weight", "offset": 25122633728, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 25135216640, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.10.attn_q_a_norm.weight", "offset": 25198131200, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.10.attn_q_a.weight", "offset": 25198137344, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.10.attn_q_b.weight", "offset": 25204035584, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 25232347136, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 25232367616, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 26176086016, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 27119804416, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 28063522816, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 28063543296, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 28063563776, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 29007282176, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 29951000576, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 30894718976, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 30894739456, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 30898016256, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 30909812736, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 30921609216, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 30933405696, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 30933407744, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.11.attn_kv_b.weight", "offset": 30935619584, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 30948202496, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.11.attn_q_a_norm.weight", "offset": 31011117056, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.11.attn_q_a.weight", "offset": 31011123200, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.11.attn_q_b.weight", "offset": 31017021440, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 31045332992, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 31045353472, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 31989071872, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 32932790272, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 33876508672, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 33876529152, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 33879805952, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 33891602432, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 33903398912, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 33915195392, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 33915197440, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.12.attn_kv_b.weight", "offset": 33917409280, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 33929992192, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.12.attn_q_a_norm.weight", "offset": 33992906752, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.12.attn_q_a.weight", "offset": 33992912896, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.12.attn_q_b.weight", "offset": 33998811136, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 34027122688, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 34027143168, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 34970861568, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 35914579968, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 36858298368, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 36858318848, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 36861595648, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 36873392128, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 36885188608, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 36896985088, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 36896987136, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.13.attn_kv_b.weight", "offset": 36899198976, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 36911781888, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.13.attn_q_a_norm.weight", "offset": 36974696448, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.13.attn_q_a.weight", "offset": 36974702592, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.13.attn_q_b.weight", "offset": 36980600832, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 37008912384, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 37008932864, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 37952651264, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 38896369664, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 39840088064, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 39840108544, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 39843385344, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 39855181824, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 39866978304, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 39878774784, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 39878776832, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.14.attn_kv_b.weight", "offset": 39880988672, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 39893571584, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.14.attn_q_a_norm.weight", "offset": 39956486144, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.14.attn_q_a.weight", "offset": 39956492288, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.14.attn_q_b.weight", "offset": 39962390528, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 39990702080, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 39990722560, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 40934440960, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 41878159360, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 42821877760, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 42821898240, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 42825175040, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 42836971520, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 42848768000, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 42860564480, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 42860566528, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.15.attn_kv_b.weight", "offset": 42862778368, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 42875361280, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.15.attn_q_a_norm.weight", "offset": 42938275840, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.15.attn_q_a.weight", "offset": 42938281984, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.15.attn_q_b.weight", "offset": 42944180224, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 42972491776, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 42972512256, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 43916230656, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.15.ffn_up_exps.weight", "offset": 44859949056, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 45803667456, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 45803687936, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 45806964736, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 45818761216, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 45830557696, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 45842354176, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 45842356224, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.16.attn_kv_b.weight", "offset": 45844568064, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 45857150976, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.16.attn_q_a_norm.weight", "offset": 45920065536, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.16.attn_q_a.weight", "offset": 45920071680, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.16.attn_q_b.weight", "offset": 45925969920, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 45954281472, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 45954301952, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 46898020352, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 47841738752, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 48785457152, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 48785477632, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 48788754432, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 48800550912, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 48812347392, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 48824143872, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 48824145920, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.17.attn_kv_b.weight", "offset": 48826357760, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 48838940672, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.17.attn_q_a_norm.weight", "offset": 48901855232, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.17.attn_q_a.weight", "offset": 48901861376, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.17.attn_q_b.weight", "offset": 48907759616, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 48936071168, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 48936091648, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 49879810048, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 50823528448, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 51767246848, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 51767267328, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 51767287808, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 52711006208, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 53654724608, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 54598443008, "shape": [ 5120, 160 ], "size": 943718400, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 54601719808, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 54613516288, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 54625312768, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 54637109248, "shape": [ 5120 ], "size": 11796480, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 54637129728, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 54637131776, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.18.attn_kv_b.weight", "offset": 54639343616, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 54651926528, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.18.attn_q_a_norm.weight", "offset": 54714841088, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.18.attn_q_a.weight", "offset": 54714847232, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.18.attn_q_b.weight", "offset": 54720745472, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 54749057024, "shape": [ 5120, 160 ], "size": 28311552, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 54752333824, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 54764130304, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 54775926784, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 54787723264, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 54787725312, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.19.attn_kv_b.weight", "offset": 54789937152, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 54802520064, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.19.attn_q_a_norm.weight", "offset": 54865434624, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.19.attn_q_a.weight", "offset": 54865440768, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.19.attn_q_b.weight", "offset": 54871339008, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 54899650560, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 54899671040, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 55843389440, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 56787107840, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 57730826240, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 57730846720, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 57734123520, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 57745920000, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 57757716480, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 57769512960, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 57769515008, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.20.attn_kv_b.weight", "offset": 57771726848, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 57784309760, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.20.attn_q_a_norm.weight", "offset": 57847224320, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.20.attn_q_a.weight", "offset": 57847230464, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.20.attn_q_b.weight", "offset": 57853128704, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 57881440256, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 57881460736, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 58825179136, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 59768897536, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 60712615936, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 60712636416, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 60715913216, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 60727709696, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 60739506176, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 60751302656, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "ight\u0001\u0000\u0000\u0000\u0000\u0002\u0000_kv_a_mqa.weight", "offset": 60751304704, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.21.attn_kv_b.weight", "offset": 60753516544, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 60766099456, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.21.attn_q_a_norm.weight", "offset": 60829014016, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.21.attn_q_a.weight", "offset": 60829020160, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.21.attn_q_b.weight", "offset": 60834918400, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 60863229952, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 60863250432, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 61806968832, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 62750687232, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 63694405632, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 63694426112, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 63697702912, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 63709499392, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 63721295872, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 63733092352, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 63733094400, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.22.attn_kv_b.weight", "offset": 63735306240, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 63747889152, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.22.attn_q_a_norm.weight", "offset": 63810803712, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.22.attn_q_a.weight", "offset": 63810809856, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.22.attn_q_b.weight", "offset": 63816708096, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 63845019648, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 63845040128, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 64788758528, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 65732476928, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 66676195328, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 66676215808, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 66679492608, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 66691289088, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 66703085568, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 66714882048, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 66714884096, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.23.attn_kv_b.weight", "offset": 66717095936, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 66729678848, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.23.attn_q_a_norm.weight", "offset": 66792593408, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.23.attn_q_a.weight", "offset": 66792599552, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.23.attn_q_b.weight", "offset": 66798497792, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 66826809344, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 66826829824, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 67770548224, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 68714266624, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 69657985024, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 69658005504, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 69661282304, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 69673078784, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 69684875264, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 69696671744, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 69696673792, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.24.attn_kv_b.weight", "offset": 69698885632, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 69711468544, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.24.attn_q_a_norm.weight", "offset": 69774383104, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.24.attn_q_a.weight", "offset": 69774389248, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.24.attn_q_b.weight", "offset": 69780287488, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 69808599040, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 69808619520, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 70752337920, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 71696056320, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 72639774720, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 72639795200, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 72643072000, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 72654868480, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 72666664960, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 72678461440, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 72678463488, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.25.attn_kv_b.weight", "offset": 72680675328, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 72693258240, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.25.attn_q_a_norm.weight", "offset": 72756172800, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.25.attn_q_a.weight", "offset": 72756178944, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.25.attn_q_b.weight", "offset": 72762077184, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 72790388736, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 72790409216, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 73734127616, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 74677846016, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 75621564416, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 75621584896, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 75624861696, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 75636658176, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 75648454656, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 75660251136, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 75660253184, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.26.attn_kv_b.weight", "offset": 75662465024, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 75675047936, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.26.attn_q_a_norm.weight", "offset": 75737962496, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.26.attn_q_a.weight", "offset": 75737968640, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.26.attn_q_b.weight", "offset": 75743866880, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 75772178432, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 75772198912, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 76715917312, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 77659635712, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 78603354112, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.27.ffn_gate_inp.weight", "offset": 78603374592, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.27.ffn_down_shexp.weight", "offset": 78606651392, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.27.ffn_gate_shexp.weight", "offset": 78618447872, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.27.ffn_up_shexp.weight", "offset": 78630244352, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.27.attn_kv_a_norm.weight", "offset": 78642040832, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.27.attn_kv_a_mqa.weight", "offset": 78642042880, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.27.attn_kv_b.weight", "offset": 78644254720, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 78656837632, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.27.attn_q_a_norm.weight", "offset": 78719752192, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.27.attn_q_a.weight", "offset": 78719758336, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.27.attn_q_b.weight", "offset": 78725656576, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 78753968128, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.27.ffn_down_exps.weight", "offset": 78753988608, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.27.ffn_gate_exps.weight", "offset": 79697707008, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.27.ffn_up_exps.weight", "offset": 80641425408, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 81585143808, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.28.ffn_gate_inp.weight", "offset": 81585164288, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.28.ffn_down_shexp.weight", "offset": 81588441088, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.28.ffn_gate_shexp.weight", "offset": 81600237568, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.28.ffn_up_shexp.weight", "offset": 81612034048, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.28.attn_kv_a_norm.weight", "offset": 81623830528, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.28.attn_kv_a_mqa.weight", "offset": 81623832576, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.28.attn_kv_b.weight", "offset": 81626044416, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 81638627328, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.28.attn_q_a_norm.weight", "offset": 81701541888, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.28.attn_q_a.weight", "offset": 81701548032, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.28.attn_q_b.weight", "offset": 81707446272, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 81735757824, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.28.ffn_down_exps.weight", "offset": 81735778304, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.28.ffn_gate_exps.weight", "offset": 82679496704, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.28.ffn_up_exps.weight", "offset": 83623215104, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.28.ffn_norm.weight", "offset": 84566933504, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.29.ffn_gate_inp.weight", "offset": 84566953984, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down_shexp.weight", "offset": 84570230784, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.29.ffn_gate_shexp.weight", "offset": 84582027264, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "hexp.weight\u0002\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000", "offset": 84593823744, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.29.attn_kv_a_norm.weight", "offset": 84605620224, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.29.attn_kv_a_mqa.weight", "offset": 84605622272, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.29.attn_kv_b.weight", "offset": 84607834112, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.29.attn_output.weight", "offset": 84620417024, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.29.attn_q_a_norm.weight", "offset": 84683331584, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.29.attn_q_a.weight", "offset": 84683337728, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.29.attn_q_b.weight", "offset": 84689235968, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.29.attn_norm.weight", "offset": 84717547520, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.29.ffn_down_exps.weight", "offset": 84717568000, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.29.ffn_gate_exps.weight", "offset": 85661286400, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.29.ffn_up_exps.weight", "offset": 86605004800, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.29.ffn_norm.weight", "offset": 87548723200, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.30.ffn_gate_inp.weight", "offset": 87548743680, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.30.ffn_down_shexp.weight", "offset": 87552020480, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.30.ffn_gate_shexp.weight", "offset": 87563816960, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.30.ffn_up_shexp.weight", "offset": 87575613440, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.30.attn_kv_a_norm.weight", "offset": 87587409920, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.30.attn_kv_a_mqa.weight", "offset": 87587411968, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.30.attn_kv_b.weight", "offset": 87589623808, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.30.attn_output.weight", "offset": 87602206720, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.30.attn_q_a_norm.weight", "offset": 87665121280, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.30.attn_q_a.weight", "offset": 87665127424, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.30.attn_q_b.weight", "offset": 87671025664, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 87699337216, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.30.ffn_down_exps.weight", "offset": 87699357696, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.30.ffn_gate_exps.weight", "offset": 88643076096, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.30.ffn_up_exps.weight", "offset": 89586794496, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.30.ffn_norm.weight", "offset": 90530512896, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 90530533376, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.31.ffn_down_exps.weight", "offset": 90530553856, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.31.ffn_gate_exps.weight", "offset": 91474272256, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.31.ffn_up_exps.weight", "offset": 92417990656, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.31.ffn_gate_inp.weight", "offset": 93361709056, "shape": [ 5120, 160 ], "size": 943718400, "type": 0 }, { "name": "blk.31.ffn_down_shexp.weight", "offset": 93364985856, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.31.ffn_gate_shexp.weight", "offset": 93376782336, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.31.ffn_up_shexp.weight", "offset": 93388578816, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.31.ffn_norm.weight", "offset": 93400375296, "shape": [ 5120 ], "size": 11796480, "type": 0 }, { "name": "blk.31.attn_kv_a_norm.weight", "offset": 93400395776, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.31.attn_kv_a_mqa.weight", "offset": 93400397824, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.31.attn_kv_b.weight", "offset": 93402609664, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.31.attn_output.weight", "offset": 93415192576, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.31.attn_q_a_norm.weight", "offset": 93478107136, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.31.attn_q_a.weight", "offset": 93478113280, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.31.attn_q_b.weight", "offset": 93484011520, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.32.ffn_gate_inp.weight", "offset": 93512323072, "shape": [ 5120, 160 ], "size": 28311552, "type": 0 }, { "name": "blk.32.ffn_down_shexp.weight", "offset": 93515599872, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.32.ffn_gate_shexp.weight", "offset": 93527396352, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.32.ffn_up_shexp.weight", "offset": 93539192832, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.32.attn_kv_a_norm.weight", "offset": 93550989312, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.32.attn_kv_a_mqa.weight", "offset": 93550991360, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.32.attn_kv_b.weight", "offset": 93553203200, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.32.attn_output.weight", "offset": 93565786112, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.32.attn_q_a_norm.weight", "offset": 93628700672, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.32.attn_q_a.weight", "offset": 93628706816, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.32.attn_q_b.weight", "offset": 93634605056, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.32.attn_norm.weight", "offset": 93662916608, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.32.ffn_down_exps.weight", "offset": 93662937088, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.32.ffn_gate_exps.weight", "offset": 94606655488, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.32.ffn_up_exps.weight", "offset": 95550373888, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.32.ffn_norm.weight", "offset": 96494092288, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.33.ffn_gate_inp.weight", "offset": 96494112768, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.33.ffn_down_shexp.weight", "offset": 96497389568, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.33.ffn_gate_shexp.weight", "offset": 96509186048, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.33.ffn_up_shexp.weight", "offset": 96520982528, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.33.attn_kv_a_norm.weight", "offset": 96532779008, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.33.attn_kv_a_mqa.weight", "offset": 96532781056, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.33.attn_kv_b.weight", "offset": 96534992896, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.33.attn_output.weight", "offset": 96547575808, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.33.attn_q_a_norm.weight", "offset": 96610490368, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.33.attn_q_a.weight", "offset": 96610496512, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.33.attn_q_b.weight", "offset": 96616394752, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.33.attn_norm.weight", "offset": 96644706304, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.33.ffn_down_exps.weight", "offset": 96644726784, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.33.ffn_gate_exps.weight", "offset": 97588445184, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.33.ffn_up_exps.weight", "offset": 98532163584, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.33.ffn_norm.weight", "offset": 99475881984, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.34.ffn_gate_inp.weight", "offset": 99475902464, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.34.ffn_down_shexp.weight", "offset": 99479179264, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.34.ffn_gate_shexp.weight", "offset": 99490975744, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.34.ffn_up_shexp.weight", "offset": 99502772224, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.34.attn_kv_a_norm.weight", "offset": 99514568704, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.34.attn_kv_a_mqa.weight", "offset": 99514570752, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.34.attn_kv_b.weight", "offset": 99516782592, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.34.attn_output.weight", "offset": 99529365504, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.34.attn_q_a_norm.weight", "offset": 99592280064, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.34.attn_q_a.weight", "offset": 99592286208, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.34.attn_q_b.weight", "offset": 99598184448, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.34.attn_norm.weight", "offset": 99626496000, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.34.ffn_down_exps.weight", "offset": 99626516480, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.34.ffn_gate_exps.weight", "offset": 100570234880, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.34.ffn_up_exps.weight", "offset": 101513953280, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.34.ffn_norm.weight", "offset": 102457671680, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.35.ffn_gate_inp.weight", "offset": 102457692160, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.35.ffn_down_shexp.weight", "offset": 102460968960, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.35.ffn_gate_shexp.weight", "offset": 102472765440, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.35.ffn_up_shexp.weight", "offset": 102484561920, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.35.attn_kv_a_norm.weight", "offset": 102496358400, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.35.attn_kv_a_mqa.weight", "offset": 102496360448, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.35.attn_kv_b.weight", "offset": 102498572288, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.35.attn_output.weight", "offset": 102511155200, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.35.attn_q_a_norm.weight", "offset": 102574069760, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.35.attn_q_a.weight", "offset": 102574075904, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.35.attn_q_b.weight", "offset": 102579974144, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.35.attn_norm.weight", "offset": 102608285696, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.35.ffn_down_exps.weight", "offset": 102608306176, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.35.ffn_gate_exps.weight", "offset": 103552024576, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.35.ffn_up_exps.weight", "offset": 104495742976, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.35.ffn_norm.weight", "offset": 105439461376, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.36.ffn_gate_inp.weight", "offset": 105439481856, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down_shexp.weight", "offset": 105442758656, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.36.ffn_gate_shexp.weight", "offset": 105454555136, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.36.ffn_up_shexp.weight", "offset": 105466351616, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.36.attn_kv_a_norm.weight", "offset": 105478148096, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.36.attn_kv_a_mqa.weight", "offset": 105478150144, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.36.attn_kv_b.weight", "offset": 105480361984, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.36.attn_output.weight", "offset": 105492944896, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.36.attn_q_a_norm.weight", "offset": 105555859456, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.36.attn_q_a.weight", "offset": 105555865600, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.36.attn_q_b.weight", "offset": 105561763840, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.36.attn_norm.weight", "offset": 105590075392, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.36.ffn_down_exps.weight", "offset": 105590095872, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.36.ffn_gate_exps.weight", "offset": 106533814272, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.36.ffn_up_exps.weight", "offset": 107477532672, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 108421251072, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.37.ffn_gate_inp.weight", "offset": 108421271552, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.37.ffn_down_shexp.weight", "offset": 108424548352, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.37.ffn_gate_shexp.weight", "offset": 108436344832, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.37.ffn_up_shexp.weight", "offset": 108448141312, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.37.attn_kv_a_norm.weight", "offset": 108459937792, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.37.attn_kv_a_mqa.weight", "offset": 108459939840, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.37.attn_kv_b.weight", "offset": 108462151680, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.37.attn_output.weight", "offset": 108474734592, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.37.attn_q_a_norm.weight", "offset": 108537649152, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.37.attn_q_a.weight", "offset": 108537655296, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.37.attn_q_b.weight", "offset": 108543553536, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.37.attn_norm.weight", "offset": 108571865088, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.37.ffn_down_exps.weight", "offset": 108571885568, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.37.ffn_gate_exps.weight", "offset": 109515603968, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.37.ffn_up_exps.weight", "offset": 110459322368, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.37.ffn_norm.weight", "offset": 111403040768, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.38.ffn_gate_inp.weight", "offset": 111403061248, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.38.ffn_down_shexp.weight", "offset": 111406338048, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.38.ffn_gate_shexp.weight", "offset": 111418134528, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.38.ffn_up_shexp.weight", "offset": 111429931008, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.38.attn_kv_a_norm.weight", "offset": 111441727488, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.38.attn_kv_a_mqa.weight", "offset": 111441729536, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.38.attn_kv_b.weight", "offset": 111443941376, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.38.attn_output.weight", "offset": 111456524288, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.38.attn_q_a_norm.weight", "offset": 111519438848, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.38.attn_q_a.weight", "offset": 111519444992, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.38.attn_q_b.weight", "offset": 111525343232, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.38.attn_norm.weight", "offset": 111553654784, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.38.ffn_down_exps.weight", "offset": 111553675264, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.38.ffn_gate_exps.weight", "offset": 112497393664, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.38.ffn_up_exps.weight", "offset": 113441112064, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.38.ffn_norm.weight", "offset": 114384830464, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.39.ffn_gate_inp.weight", "offset": 114384850944, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.39.ffn_down_shexp.weight", "offset": 114388127744, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.39.ffn_gate_shexp.weight", "offset": 114399924224, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.39.ffn_up_shexp.weight", "offset": 114411720704, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.39.attn_kv_a_norm.weight", "offset": 114423517184, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.39.attn_kv_a_mqa.weight", "offset": 114423519232, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.39.attn_kv_b.weight", "offset": 114425731072, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.39.attn_output.weight", "offset": 114438313984, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.39.attn_q_a_norm.weight", "offset": 114501228544, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.39.attn_q_a.weight", "offset": 114501234688, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.39.attn_q_b.weight", "offset": 114507132928, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.39.attn_norm.weight", "offset": 114535444480, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.39.ffn_down_exps.weight", "offset": 114535464960, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.39.ffn_gate_exps.weight", "offset": 115479183360, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.39.ffn_up_exps.weight", "offset": 116422901760, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.39.ffn_norm.weight", "offset": 117366620160, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.40.ffn_gate_inp.weight", "offset": 117366640640, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.40.ffn_down_shexp.weight", "offset": 117369917440, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.40.ffn_gate_shexp.weight", "offset": 117381713920, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.40.ffn_up_shexp.weight", "offset": 117393510400, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.40.attn_kv_a_norm.weight", "offset": 117405306880, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.40.attn_kv_a_mqa.weight", "offset": 117405308928, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.40.attn_kv_b.weight", "offset": 117407520768, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.40.attn_output.weight", "offset": 117420103680, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.40.attn_q_a_norm.weight", "offset": 117483018240, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.40.attn_q_a.weight", "offset": 117483024384, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.40.attn_q_b.weight", "offset": 117488922624, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.40.attn_norm.weight", "offset": 117517234176, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.40.ffn_down_exps.weight", "offset": 117517254656, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.40.ffn_gate_exps.weight", "offset": 118460973056, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.40.ffn_up_exps.weight", "offset": 119404691456, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.40.ffn_norm.weight", "offset": 120348409856, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.41.ffn_gate_inp.weight", "offset": 120348430336, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.41.ffn_down_shexp.weight", "offset": 120351707136, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.41.ffn_gate_shexp.weight", "offset": 120363503616, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.41.ffn_up_shexp.weight", "offset": 120375300096, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.41.attn_kv_a_norm.weight", "offset": 120387096576, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.41.attn_kv_a_mqa.weight", "offset": 120387098624, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.41.attn_kv_b.weight", "offset": 120389310464, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.41.attn_output.weight", "offset": 120401893376, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.41.attn_q_a_norm.weight", "offset": 120464807936, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.41.attn_q_a.weight", "offset": 120464814080, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.41.attn_q_b.weight", "offset": 120470712320, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.41.attn_norm.weight", "offset": 120499023872, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.41.ffn_down_exps.weight", "offset": 120499044352, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.41.ffn_gate_exps.weight", "offset": 121442762752, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.41.ffn_up_exps.weight", "offset": 122386481152, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.41.ffn_norm.weight", "offset": 123330199552, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.42.ffn_gate_inp.weight", "offset": 123330220032, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.42.ffn_down_shexp.weight", "offset": 123333496832, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.42.ffn_gate_shexp.weight", "offset": 123345293312, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.42.ffn_up_shexp.weight", "offset": 123357089792, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.42.attn_kv_a_norm.weight", "offset": 123368886272, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.42.attn_kv_a_mqa.weight", "offset": 123368888320, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.42.attn_kv_b.weight", "offset": 123371100160, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.42.attn_output.weight", "offset": 123383683072, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.42.attn_q_a_norm.weight", "offset": 123446597632, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.42.attn_q_a.weight", "offset": 123446603776, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.42.attn_q_b.weight", "offset": 123452502016, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.42.attn_norm.weight", "offset": 123480813568, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.42.ffn_down_exps.weight", "offset": 123480834048, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.42.ffn_gate_exps.weight", "offset": 124424552448, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.42.ffn_up_exps.weight", "offset": 125368270848, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.42.ffn_norm.weight", "offset": 126311989248, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.43.ffn_gate_inp.weight", "offset": 126312009728, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.43.ffn_down_shexp.weight", "offset": 126315286528, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.43.ffn_gate_shexp.weight", "offset": 126327083008, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.43.ffn_up_shexp.weight", "offset": 126338879488, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.43.attn_kv_a_norm.weight", "offset": 126350675968, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.43.attn_kv_a_mqa.weight", "offset": 126350678016, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.43.attn_kv_b.weight", "offset": 126352889856, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.43.attn_output.weight", "offset": 126365472768, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.43.attn_q_a_norm.weight", "offset": 126428387328, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.43.attn_q_a.weight", "offset": 126428393472, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.43.attn_q_b.weight", "offset": 126434291712, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.43.attn_norm.weight", "offset": 126462603264, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.43.ffn_down_exps.weight", "offset": 126462623744, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.43.ffn_gate_exps.weight", "offset": 127406342144, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.43.ffn_up_exps.weight", "offset": 128350060544, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.43.ffn_norm.weight", "offset": 129293778944, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.44.attn_norm.weight", "offset": 129293799424, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.44.ffn_down_exps.weight", "offset": 129293819904, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.44.ffn_gate_exps.weight", "offset": 130237538304, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.44.ffn_up_exps.weight", "offset": 131181256704, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.44.ffn_gate_inp.weight", "offset": 132124975104, "shape": [ 5120, 160 ], "size": 943718400, "type": 0 }, { "name": "blk.44.ffn_down_shexp.weight", "offset": 132128251904, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.44.ffn_gate_shexp.weight", "offset": 132140048384, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.44.ffn_up_shexp.weight", "offset": 132151844864, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.44.ffn_norm.weight", "offset": 132163641344, "shape": [ 5120 ], "size": 11796480, "type": 0 }, { "name": "blk.44.attn_kv_a_norm.weight", "offset": 132163661824, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.44.attn_kv_a_mqa.weight", "offset": 132163663872, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.44.attn_kv_b.weight", "offset": 132165875712, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.44.attn_output.weight", "offset": 132178458624, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.44.attn_q_a_norm.weight", "offset": 132241373184, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.44.attn_q_a.weight", "offset": 132241379328, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.44.attn_q_b.weight", "offset": 132247277568, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.45.ffn_gate_inp.weight", "offset": 132275589120, "shape": [ 5120, 160 ], "size": 28311552, "type": 0 }, { "name": "blk.45.ffn_down_shexp.weight", "offset": 132278865920, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.45.ffn_gate_shexp.weight", "offset": 132290662400, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.45.ffn_up_shexp.weight", "offset": 132302458880, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.45.attn_kv_a_norm.weight", "offset": 132314255360, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.45.attn_kv_a_mqa.weight", "offset": 132314257408, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.45.attn_kv_b.weight", "offset": 132316469248, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.45.attn_output.weight", "offset": 132329052160, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.45.attn_q_a_norm.weight", "offset": 132391966720, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.45.attn_q_a.weight", "offset": 132391972864, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.45.attn_q_b.weight", "offset": 132397871104, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.45.attn_norm.weight", "offset": 132426182656, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.45.ffn_down_exps.weight", "offset": 132426203136, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.45.ffn_gate_exps.weight", "offset": 133369921536, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.45.ffn_up_exps.weight", "offset": 134313639936, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.45.ffn_norm.weight", "offset": 135257358336, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.46.ffn_gate_inp.weight", "offset": 135257378816, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.46.ffn_down_shexp.weight", "offset": 135260655616, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.46.ffn_gate_shexp.weight", "offset": 135272452096, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.46.ffn_up_shexp.weight", "offset": 135284248576, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.46.attn_kv_a_norm.weight", "offset": 135296045056, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.46.attn_kv_a_mqa.weight", "offset": 135296047104, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.46.attn_kv_b.weight", "offset": 135298258944, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.46.attn_output.weight", "offset": 135310841856, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.46.attn_q_a_norm.weight", "offset": 135373756416, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.46.attn_q_a.weight", "offset": 135373762560, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.46.attn_q_b.weight", "offset": 135379660800, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.46.attn_norm.weight", "offset": 135407972352, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.46.ffn_down_exps.weight", "offset": 135407992832, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.46.ffn_gate_exps.weight", "offset": 136351711232, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.46.ffn_up_exps.weight", "offset": 137295429632, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.46.ffn_norm.weight", "offset": 138239148032, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.47.ffn_gate_inp.weight", "offset": 138239168512, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.47.ffn_down_shexp.weight", "offset": 138242445312, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.47.ffn_gate_shexp.weight", "offset": 138254241792, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.47.ffn_up_shexp.weight", "offset": 138266038272, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.47.attn_kv_a_norm.weight", "offset": 138277834752, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.47.attn_kv_a_mqa.weight", "offset": 138277836800, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.47.attn_kv_b.weight", "offset": 138280048640, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.47.attn_output.weight", "offset": 138292631552, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.47.attn_q_a_norm.weight", "offset": 138355546112, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.47.attn_q_a.weight", "offset": 138355552256, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.47.attn_q_b.weight", "offset": 138361450496, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.47.attn_norm.weight", "offset": 138389762048, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.47.ffn_down_exps.weight", "offset": 138389782528, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.47.ffn_gate_exps.weight", "offset": 139333500928, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.47.ffn_up_exps.weight", "offset": 140277219328, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.47.ffn_norm.weight", "offset": 141220937728, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.48.ffn_gate_inp.weight", "offset": 141220958208, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.48.ffn_down_shexp.weight", "offset": 141224235008, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.48.ffn_gate_shexp.weight", "offset": 141236031488, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.48.ffn_up_shexp.weight", "offset": 141247827968, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.48.attn_kv_a_norm.weight", "offset": 141259624448, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.48.attn_kv_a_mqa.weight", "offset": 141259626496, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.48.attn_kv_b.weight", "offset": 141261838336, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.48.attn_output.weight", "offset": 141274421248, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.48.attn_q_a_norm.weight", "offset": 141337335808, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.48.attn_q_a.weight", "offset": 141337341952, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.48.attn_q_b.weight", "offset": 141343240192, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.48.attn_norm.weight", "offset": 141371551744, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.48.ffn_down_exps.weight", "offset": 141371572224, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.48.ffn_gate_exps.weight", "offset": 142315290624, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.48.ffn_up_exps.weight", "offset": 143259009024, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.48.ffn_norm.weight", "offset": 144202727424, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.49.ffn_gate_inp.weight", "offset": 144202747904, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.49.ffn_down_shexp.weight", "offset": 144206024704, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.49.ffn_gate_shexp.weight", "offset": 144217821184, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.49.ffn_up_shexp.weight", "offset": 144229617664, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.49.attn_kv_a_norm.weight", "offset": 144241414144, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.49.attn_kv_a_mqa.weight", "offset": 144241416192, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.49.attn_kv_b.weight", "offset": 144243628032, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.49.attn_output.weight", "offset": 144256210944, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.49.attn_q_a_norm.weight", "offset": 144319125504, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.49.attn_q_a.weight", "offset": 144319131648, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.49.attn_q_b.weight", "offset": 144325029888, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.49.attn_norm.weight", "offset": 144353341440, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.49.ffn_down_exps.weight", "offset": 144353361920, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.49.ffn_gate_exps.weight", "offset": 145297080320, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.49.ffn_up_exps.weight", "offset": 146240798720, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.49.ffn_norm.weight", "offset": 147184517120, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.50.ffn_gate_inp.weight", "offset": 147184537600, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.50.ffn_down_shexp.weight", "offset": 147187814400, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.50.ffn_gate_shexp.weight", "offset": 147199610880, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.50.ffn_up_shexp.weight", "offset": 147211407360, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.50.attn_kv_a_norm.weight", "offset": 147223203840, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.50.attn_kv_a_mqa.weight", "offset": 147223205888, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.50.attn_kv_b.weight", "offset": 147225417728, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.50.attn_output.weight", "offset": 147238000640, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.50.attn_q_a_norm.weight", "offset": 147300915200, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.50.attn_q_a.weight", "offset": 147300921344, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.50.attn_q_b.weight", "offset": 147306819584, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.50.attn_norm.weight", "offset": 147335131136, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.50.ffn_down_exps.weight", "offset": 147335151616, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.50.ffn_gate_exps.weight", "offset": 148278870016, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.50.ffn_up_exps.weight", "offset": 149222588416, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.50.ffn_norm.weight", "offset": 150166306816, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.51.ffn_gate_inp.weight", "offset": 150166327296, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.51.ffn_down_shexp.weight", "offset": 150169604096, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.51.ffn_gate_shexp.weight", "offset": 150181400576, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.51.ffn_up_shexp.weight", "offset": 150193197056, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.51.attn_kv_a_norm.weight", "offset": 150204993536, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.51.attn_kv_a_mqa.weight", "offset": 150204995584, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.51.attn_kv_b.weight", "offset": 150207207424, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.51.attn_output.weight", "offset": 150219790336, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.51.attn_q_a_norm.weight", "offset": 150282704896, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.51.attn_q_a.weight", "offset": 150282711040, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.51.attn_q_b.weight", "offset": 150288609280, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.51.attn_norm.weight", "offset": 150316920832, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.51.ffn_down_exps.weight", "offset": 150316941312, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.51.ffn_gate_exps.weight", "offset": 151260659712, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.51.ffn_up_exps.weight", "offset": 152204378112, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.51.ffn_norm.weight", "offset": 153148096512, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.52.ffn_gate_inp.weight", "offset": 153148116992, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.52.ffn_down_shexp.weight", "offset": 153151393792, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.52.ffn_gate_shexp.weight", "offset": 153163190272, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.52.ffn_up_shexp.weight", "offset": 153174986752, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.52.attn_kv_a_norm.weight", "offset": 153186783232, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.52.attn_kv_a_mqa.weight", "offset": 153186785280, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.52.attn_kv_b.weight", "offset": 153188997120, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.52.attn_output.weight", "offset": 153201580032, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.52.attn_q_a_norm.weight", "offset": 153264494592, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.52.attn_q_a.weight", "offset": 153264500736, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.52.attn_q_b.weight", "offset": 153270398976, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.52.attn_norm.weight", "offset": 153298710528, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.52.ffn_down_exps.weight", "offset": 153298731008, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.52.ffn_gate_exps.weight", "offset": 154242449408, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.52.ffn_up_exps.weight", "offset": 155186167808, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.52.ffn_norm.weight", "offset": 156129886208, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.53.ffn_gate_inp.weight", "offset": 156129906688, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.53.ffn_down_shexp.weight", "offset": 156133183488, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.53.ffn_gate_shexp.weight", "offset": 156144979968, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.53.ffn_up_shexp.weight", "offset": 156156776448, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.53.attn_kv_a_norm.weight", "offset": 156168572928, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.53.attn_kv_a_mqa.weight", "offset": 156168574976, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.53.attn_kv_b.weight", "offset": 156170786816, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.53.attn_output.weight", "offset": 156183369728, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.53.attn_q_a_norm.weight", "offset": 156246284288, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.53.attn_q_a.weight", "offset": 156246290432, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.53.attn_q_b.weight", "offset": 156252188672, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.53.attn_norm.weight", "offset": 156280500224, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.53.ffn_down_exps.weight", "offset": 156280520704, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.53.ffn_gate_exps.weight", "offset": 157224239104, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.53.ffn_up_exps.weight", "offset": 158167957504, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.53.ffn_norm.weight", "offset": 159111675904, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.54.ffn_gate_inp.weight", "offset": 159111696384, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.54.ffn_down_shexp.weight", "offset": 159114973184, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.54.ffn_gate_shexp.weight", "offset": 159126769664, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.54.ffn_up_shexp.weight", "offset": 159138566144, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.54.attn_kv_a_norm.weight", "offset": 159150362624, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.54.attn_kv_a_mqa.weight", "offset": 159150364672, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.54.attn_kv_b.weight", "offset": 159152576512, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.54.attn_output.weight", "offset": 159165159424, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.54.attn_q_a_norm.weight", "offset": 159228073984, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.54.attn_q_a.weight", "offset": 159228080128, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.54.attn_q_b.weight", "offset": 159233978368, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.54.attn_norm.weight", "offset": 159262289920, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.54.ffn_down_exps.weight", "offset": 159262310400, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.54.ffn_gate_exps.weight", "offset": 160206028800, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.54.ffn_up_exps.weight", "offset": 161149747200, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.54.ffn_norm.weight", "offset": 162093465600, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.55.ffn_gate_inp.weight", "offset": 162093486080, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.55.ffn_down_shexp.weight", "offset": 162096762880, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.55.ffn_gate_shexp.weight", "offset": 162108559360, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.55.ffn_up_shexp.weight", "offset": 162120355840, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.55.attn_kv_a_norm.weight", "offset": 162132152320, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.55.attn_kv_a_mqa.weight", "offset": 162132154368, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.55.attn_kv_b.weight", "offset": 162134366208, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.55.attn_output.weight", "offset": 162146949120, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.55.attn_q_a_norm.weight", "offset": 162209863680, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.55.attn_q_a.weight", "offset": 162209869824, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.55.attn_q_b.weight", "offset": 162215768064, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.55.attn_norm.weight", "offset": 162244079616, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.55.ffn_down_exps.weight", "offset": 162244100096, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.55.ffn_gate_exps.weight", "offset": 163187818496, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.55.ffn_up_exps.weight", "offset": 164131536896, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.55.ffn_norm.weight", "offset": 165075255296, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.56.ffn_gate_inp.weight", "offset": 165075275776, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.56.ffn_down_shexp.weight", "offset": 165078552576, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.56.ffn_gate_shexp.weight", "offset": 165090349056, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.56.ffn_up_shexp.weight", "offset": 165102145536, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.56.attn_kv_a_norm.weight", "offset": 165113942016, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.56.attn_kv_a_mqa.weight", "offset": 165113944064, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.56.attn_kv_b.weight", "offset": 165116155904, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.56.attn_output.weight", "offset": 165128738816, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.56.attn_q_a_norm.weight", "offset": 165191653376, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.56.attn_q_a.weight", "offset": 165191659520, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.56.attn_q_b.weight", "offset": 165197557760, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.56.attn_norm.weight", "offset": 165225869312, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.56.ffn_down_exps.weight", "offset": 165225889792, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.56.ffn_gate_exps.weight", "offset": 166169608192, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.56.ffn_up_exps.weight", "offset": 167113326592, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.56.ffn_norm.weight", "offset": 168057044992, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.57.attn_norm.weight", "offset": 168057065472, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.57.ffn_down_exps.weight", "offset": 168057085952, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.57.ffn_gate_exps.weight", "offset": 169000804352, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.57.ffn_up_exps.weight", "offset": 169944522752, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.57.ffn_gate_inp.weight", "offset": 170888241152, "shape": [ 5120, 160 ], "size": 943718400, "type": 0 }, { "name": "blk.57.ffn_down_shexp.weight", "offset": 170891517952, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.57.ffn_gate_shexp.weight", "offset": 170903314432, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.57.ffn_up_shexp.weight", "offset": 170915110912, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.57.ffn_norm.weight", "offset": 170926907392, "shape": [ 5120 ], "size": 11796480, "type": 0 }, { "name": "blk.57.attn_kv_a_norm.weight", "offset": 170926927872, "shape": [ 512 ], "size": 20480, "type": 0 }, { "name": "blk.57.attn_kv_a_mqa.weight", "offset": 170926929920, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.57.attn_kv_b.weight", "offset": 170929141760, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.57.attn_output.weight", "offset": 170941724672, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.57.attn_q_a_norm.weight", "offset": 171004639232, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.57.attn_q_a.weight", "offset": 171004645376, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.57.attn_q_b.weight", "offset": 171010543616, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.58.ffn_gate_inp.weight", "offset": 171038855168, "shape": [ 5120, 160 ], "size": 28311552, "type": 0 }, { "name": "blk.58.ffn_down_shexp.weight", "offset": 171042131968, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.58.ffn_gate_shexp.weight", "offset": 171053928448, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.58.ffn_up_shexp.weight", "offset": 171065724928, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.58.attn_kv_a_norm.weight", "offset": 171077521408, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.58.attn_kv_a_mqa.weight", "offset": 171077523456, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.58.attn_kv_b.weight", "offset": 171079735296, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.58.attn_output.weight", "offset": 171092318208, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.58.attn_q_a_norm.weight", "offset": 171155232768, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.58.attn_q_a.weight", "offset": 171155238912, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.58.attn_q_b.weight", "offset": 171161137152, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.58.attn_norm.weight", "offset": 171189448704, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.58.ffn_down_exps.weight", "offset": 171189469184, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.58.ffn_gate_exps.weight", "offset": 172133187584, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.58.ffn_up_exps.weight", "offset": 173076905984, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.58.ffn_norm.weight", "offset": 174020624384, "shape": [ 5120 ], "size": 943718400, "type": 0 }, { "name": "blk.59.ffn_gate_inp.weight", "offset": 174020644864, "shape": [ 5120, 160 ], "size": 20480, "type": 0 }, { "name": "blk.59.ffn_down_shexp.weight", "offset": 174023921664, "shape": [ 3072, 5120 ], "size": 3276800, "type": 7 }, { "name": "blk.59.ffn_gate_shexp.weight", "offset": 174035718144, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.59.ffn_up_shexp.weight", "offset": 174047514624, "shape": [ 5120, 3072 ], "size": 11796480, "type": 7 }, { "name": "blk.59.attn_kv_a_norm.weight", "offset": 174059311104, "shape": [ 512 ], "size": 11796480, "type": 0 }, { "name": "blk.59.attn_kv_a_mqa.weight", "offset": 174059313152, "shape": [ 5120, 576 ], "size": 2048, "type": 7 }, { "name": "blk.59.attn_kv_b.weight", "offset": 174061524992, "shape": [ 512, 32768 ], "size": 2211840, "type": 7 }, { "name": "blk.59.attn_output.weight", "offset": 174074107904, "shape": [ 16384, 5120 ], "size": 12582912, "type": 7 }, { "name": "blk.59.attn_q_a_norm.weight", "offset": 174137022464, "shape": [ 1536 ], "size": 62914560, "type": 0 }, { "name": "blk.59.attn_q_a.weight", "offset": 174137028608, "shape": [ 5120, 1536 ], "size": 6144, "type": 7 }, { "name": "blk.59.attn_q_b.weight", "offset": 174142926848, "shape": [ 1536, 24576 ], "size": 5898240, "type": 7 }, { "name": "blk.59.attn_norm.weight", "offset": 174171238400, "shape": [ 5120 ], "size": 28311552, "type": 0 }, { "name": "blk.59.ffn_down_exps.weight", "offset": 174171258880, "shape": [ 1536, 5120, 160 ], "size": 20480, "type": 7 }, { "name": "blk.59.ffn_gate_exps.weight", "offset": 175114977280, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.59.ffn_up_exps.weight", "offset": 176058695680, "shape": [ 5120, 1536, 160 ], "size": 943718400, "type": 7 }, { "name": "blk.59.ffn_norm.weight", "offset": 177002414080, "shape": [ 5120 ], "size": 943718400, "type": 0 } ], "version": 3 }