deepseek-coder-v2

{ "metadata": { "deepseek2.attention.head_count": 16, "deepseek2.attention.head_count_kv": 16, "deepseek2.attention.key_length": 192, "deepseek2.attention.kv_lora_rank": 512, "deepseek2.attention.layer_norm_rms_epsilon": 0.000001, "deepseek2.attention.value_length": 128, "deepseek2.block_count": 27, "deepseek2.context_length": 163840, "deepseek2.embedding_length": 2048, "deepseek2.expert_count": 64, "deepseek2.expert_feed_forward_length": 1408, "deepseek2.expert_shared_count": 2, "deepseek2.expert_used_count": 6, "deepseek2.expert_weights_scale": 1, "deepseek2.feed_forward_length": 10944, "deepseek2.leading_dense_block_count": 1, "deepseek2.rope.dimension_count": 64, "deepseek2.rope.freq_base": 10000, "deepseek2.rope.scaling.factor": 40, "deepseek2.rope.scaling.original_context_length": 4096, "deepseek2.rope.scaling.type": "yarn", "deepseek2.rope.scaling.yarn_log_multiplier": 0.0707, "deepseek2.vocab_size": 102400, "general.architecture": "deepseek2", "general.file_type": 8, "general.name": "DeepSeek-Coder-V2-Lite-Instruct", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 100000, "tokenizer.ggml.eos_token_id": 100001, "tokenizer.ggml.merges": "... (99757 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 100001, "tokenizer.ggml.pre": "deepseek-llm", "tokenizer.ggml.token_type": "... (102400 values)", "tokenizer.ggml.tokens": "... (102400 values)" }, "num_params": 15706484224, "tensors": [ { "name": "token_embd.weight", "offset": 172032000, "shape": [ 2048, 102400 ], "size": 172032000, "type": 6 }, { "name": "blk.0.attn_norm.weight", "offset": 316211200, "shape": [ 2048 ], "size": 144179200, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 316219392, "shape": [ 10944, 2048 ], "size": 8192, "type": 6 }, { "name": "blk.0.ffn_gate.weight", "offset": 331628544, "shape": [ 2048, 10944 ], "size": 15409152, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 347037696, "shape": [ 2048, 10944 ], "size": 15409152, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 362446848, "shape": [ 2048 ], "size": 15409152, "type": 0 }, { "name": "blk.0.attn_kv_a_norm.weight", "offset": 362455040, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.0.attn_kv_a_mqa.weight", "offset": 362457088, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.0.attn_kv_b.weight", "offset": 363268096, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 364709888, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.0.attn_q.weight", "offset": 367593472, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 371918848, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.1.ffn_down_exps.weight", "offset": 371927040, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.1.ffn_gate_exps.weight", "offset": 498804736, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.1.ffn_up_exps.weight", "offset": 625682432, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.1.ffn_gate_inp.weight", "offset": 752560128, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.1.ffn_down_shexp.weight", "offset": 753084416, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.1.ffn_gate_shexp.weight", "offset": 757049344, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.1.ffn_up_shexp.weight", "offset": 761014272, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 764979200, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.1.attn_kv_a_norm.weight", "offset": 764987392, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.1.attn_kv_a_mqa.weight", "offset": 764989440, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.1.attn_kv_b.weight", "offset": 765800448, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 767242240, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.1.attn_q.weight", "offset": 770125824, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 774451200, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.2.ffn_down_exps.weight", "offset": 774459392, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.2.ffn_gate_exps.weight", "offset": 901337088, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.2.ffn_up_exps.weight", "offset": 1028214784, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.2.ffn_gate_inp.weight", "offset": 1155092480, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.2.ffn_down_shexp.weight", "offset": 1155616768, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.2.ffn_gate_shexp.weight", "offset": 1159581696, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.2.ffn_up_shexp.weight", "offset": 1163546624, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 1167511552, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.2.attn_kv_a_norm.weight", "offset": 1167519744, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.2.attn_kv_a_mqa.weight", "offset": 1167521792, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.2.attn_kv_b.weight", "offset": 1168332800, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 1169774592, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.2.attn_q.weight", "offset": 1172658176, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 1176983552, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.3.ffn_down_exps.weight", "offset": 1176991744, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.3.ffn_gate_exps.weight", "offset": 1303869440, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.3.ffn_up_exps.weight", "offset": 1430747136, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.3.ffn_gate_inp.weight", "offset": 1557624832, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.3.ffn_down_shexp.weight", "offset": 1558149120, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.3.ffn_gate_shexp.weight", "offset": 1562114048, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.3.ffn_up_shexp.weight", "offset": 1566078976, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 1570043904, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.3.attn_kv_a_norm.weight", "offset": 1570052096, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.3.attn_kv_a_mqa.weight", "offset": 1570054144, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.3.attn_kv_b.weight", "offset": 1570865152, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 1572306944, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.3.attn_q.weight", "offset": 1575190528, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 1579515904, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.4.ffn_down_exps.weight", "offset": 1579524096, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.4.ffn_gate_exps.weight", "offset": 1706401792, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.4.ffn_up_exps.weight", "offset": 1833279488, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.4.ffn_gate_inp.weight", "offset": 1960157184, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.4.ffn_down_shexp.weight", "offset": 1960681472, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.4.ffn_gate_shexp.weight", "offset": 1964646400, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.4.ffn_up_shexp.weight", "offset": 1968611328, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 1972576256, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.4.attn_kv_a_norm.weight", "offset": 1972584448, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.4.attn_kv_a_mqa.weight", "offset": 1972586496, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.4.attn_kv_b.weight", "offset": 1973397504, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 1974839296, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.4.attn_q.weight", "offset": 1977722880, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 1982048256, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.5.ffn_down_exps.weight", "offset": 1982056448, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.5.ffn_gate_exps.weight", "offset": 2108934144, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.5.ffn_up_exps.weight", "offset": 2235811840, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.5.ffn_gate_inp.weight", "offset": 2362689536, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.5.ffn_down_shexp.weight", "offset": 2363213824, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.5.ffn_gate_shexp.weight", "offset": 2367178752, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.5.ffn_up_shexp.weight", "offset": 2371143680, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 2375108608, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.5.attn_kv_a_norm.weight", "offset": 2375116800, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.5.attn_kv_a_mqa.weight", "offset": 2375118848, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.5.attn_kv_b.weight", "offset": 2375929856, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 2377371648, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.5.attn_q.weight", "offset": 2380255232, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 2384580608, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.6.ffn_down_exps.weight", "offset": 2384588800, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.6.ffn_gate_exps.weight", "offset": 2511466496, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.6.ffn_up_exps.weight", "offset": 2638344192, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.6.ffn_gate_inp.weight", "offset": 2765221888, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.6.ffn_down_shexp.weight", "offset": 2765746176, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.6.ffn_gate_shexp.weight", "offset": 2769711104, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.6.ffn_up_shexp.weight", "offset": 2773676032, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 2777640960, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.6.attn_kv_a_norm.weight", "offset": 2777649152, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.6.attn_kv_a_mqa.weight", "offset": 2777651200, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.6.attn_kv_b.weight", "offset": 2778462208, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 2779904000, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.6.attn_q.weight", "offset": 2782787584, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.7.ffn_gate_inp.weight", "offset": 2787112960, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.7.ffn_down_shexp.weight", "offset": 2787637248, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.7.ffn_gate_shexp.weight", "offset": 2791602176, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.7.ffn_up_shexp.weight", "offset": 2795567104, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.7.attn_kv_a_norm.weight", "offset": 2799532032, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.7.attn_kv_a_mqa.weight", "offset": 2799534080, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.7.attn_kv_b.weight", "offset": 2800345088, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 2801786880, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.7.attn_q.weight", "offset": 2804670464, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "output_norm.weight", "offset": 2808995840, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.10.attn_norm.weight", "offset": 2809004032, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.10.ffn_down_exps.weight", "offset": 2809012224, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.10.ffn_gate_exps.weight", "offset": 2935889920, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.10.ffn_up_exps.weight", "offset": 3062767616, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.10.ffn_gate_inp.weight", "offset": 3189645312, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.10.ffn_down_shexp.weight", "offset": 3190169600, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.10.ffn_gate_shexp.weight", "offset": 3194134528, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.10.ffn_up_shexp.weight", "offset": 3198099456, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 3202064384, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.10.attn_kv_a_norm.weight", "offset": 3202072576, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.10.attn_kv_a_mqa.weight", "offset": 3202074624, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.10.attn_kv_b.weight", "offset": 3202885632, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 3204327424, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.10.attn_q.weight", "offset": 3207211008, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 3211536384, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.11.ffn_down_exps.weight", "offset": 3211544576, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.11.ffn_gate_exps.weight", "offset": 3338422272, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.11.ffn_up_exps.weight", "offset": 3465299968, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.11.ffn_gate_inp.weight", "offset": 3592177664, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.11.ffn_down_shexp.weight", "offset": 3592701952, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.11.ffn_gate_shexp.weight", "offset": 3596666880, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.11.ffn_up_shexp.weight", "offset": 3600631808, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 3604596736, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.11.attn_kv_a_norm.weight", "offset": 3604604928, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.11.attn_kv_a_mqa.weight", "offset": 3604606976, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.11.attn_kv_b.weight", "offset": 3605417984, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 3606859776, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.11.attn_q.weight", "offset": 3609743360, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 3614068736, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.12.ffn_down_exps.weight", "offset": 3614076928, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.12.ffn_gate_exps.weight", "offset": 3740954624, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.12.ffn_up_exps.weight", "offset": 3867832320, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.12.ffn_gate_inp.weight", "offset": 3994710016, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.12.ffn_down_shexp.weight", "offset": 3995234304, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.12.ffn_gate_shexp.weight", "offset": 3999199232, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.12.ffn_up_shexp.weight", "offset": 4003164160, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 4007129088, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.12.attn_kv_a_norm.weight", "offset": 4007137280, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.12.attn_kv_a_mqa.weight", "offset": 4007139328, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.12.attn_kv_b.weight", "offset": 4007950336, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 4009392128, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.12.attn_q.weight", "offset": 4012275712, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 4016601088, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.13.ffn_down_exps.weight", "offset": 4016609280, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.13.ffn_gate_exps.weight", "offset": 4143486976, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.13.ffn_up_exps.weight", "offset": 4270364672, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.13.ffn_gate_inp.weight", "offset": 4397242368, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.13.ffn_down_shexp.weight", "offset": 4397766656, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.13.ffn_gate_shexp.weight", "offset": 4401731584, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.13.ffn_up_shexp.weight", "offset": 4405696512, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 4409661440, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.13.attn_kv_a_norm.weight", "offset": 4409669632, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.13.attn_kv_a_mqa.weight", "offset": 4409671680, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.13.attn_kv_b.weight", "offset": 4410482688, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 4411924480, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.13.attn_q.weight", "offset": 4414808064, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.14.ffn_gate_inp.weight", "offset": 4419133440, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down_shexp.weight", "offset": 4419657728, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.14.ffn_gate_shexp.weight", "offset": 4423622656, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.14.ffn_up_shexp.weight", "offset": 4427587584, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.14.attn_kv_a_norm.weight", "offset": 4431552512, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.14.attn_kv_a_mqa.weight", "offset": 4431554560, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.14.attn_kv_b.weight", "offset": 4432365568, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 4433807360, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.14.attn_q.weight", "offset": 4436690944, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.7.attn_norm.weight", "offset": 4441016320, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.7.ffn_down_exps.weight", "offset": 4441024512, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.7.ffn_gate_exps.weight", "offset": 4567902208, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.7.ffn_up_exps.weight", "offset": 4694779904, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 4821657600, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 4821665792, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.8.ffn_down_exps.weight", "offset": 4821673984, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.8.ffn_gate_exps.weight", "offset": 4948551680, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.8.ffn_up_exps.weight", "offset": 5075429376, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.8.ffn_gate_inp.weight", "offset": 5202307072, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.8.ffn_down_shexp.weight", "offset": 5202831360, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.8.ffn_gate_shexp.weight", "offset": 5206796288, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.8.ffn_up_shexp.weight", "offset": 5210761216, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 5214726144, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.8.attn_kv_a_norm.weight", "offset": 5214734336, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.8.attn_kv_a_mqa.weight", "offset": 5214736384, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.8.attn_kv_b.weight", "offset": 5215547392, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 5216989184, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.8.attn_q.weight", "offset": 5219872768, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 5224198144, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.9.ffn_down_exps.weight", "offset": 5224206336, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.9.ffn_gate_exps.weight", "offset": 5351084032, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.9.ffn_up_exps.weight", "offset": 5477961728, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.9.ffn_gate_inp.weight", "offset": 5604839424, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.9.ffn_down_shexp.weight", "offset": 5605363712, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.9.ffn_gate_shexp.weight", "offset": 5609328640, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.9.ffn_up_shexp.weight", "offset": 5613293568, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 5617258496, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.9.attn_kv_a_norm.weight", "offset": 5617266688, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.9.attn_kv_a_mqa.weight", "offset": 5617268736, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.9.attn_kv_b.weight", "offset": 5618079744, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 5619521536, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.9.attn_q.weight", "offset": 5622405120, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 5626730496, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.14.ffn_down_exps.weight", "offset": 5626738688, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.14.ffn_gate_exps.weight", "offset": 5753616384, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.14.ffn_up_exps.weight", "offset": 5880494080, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 6007371776, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 6007379968, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.15.ffn_down_exps.weight", "offset": 6007388160, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.15.ffn_gate_exps.weight", "offset": 6134265856, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "\u0000\ufffdw\ufffd\u0001\u0000\u0000\u0000\u001b\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.19.at", "offset": 6261143552, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.15.ffn_gate_inp.weight", "offset": 6388021248, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.15.ffn_down_shexp.weight", "offset": 6388545536, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.15.ffn_gate_shexp.weight", "offset": 6392510464, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.15.ffn_up_shexp.weight", "offset": 6396475392, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 6400440320, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.15.attn_kv_a_norm.weight", "offset": 6400448512, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.15.attn_kv_a_mqa.weight", "offset": 6400450560, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.15.attn_kv_b.weight", "offset": 6401261568, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 6402703360, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.15.attn_q.weight", "offset": 6405586944, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 6409912320, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.16.ffn_down_exps.weight", "offset": 6409920512, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.16.ffn_gate_exps.weight", "offset": 6536798208, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.16.ffn_up_exps.weight", "offset": 6663675904, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.16.ffn_gate_inp.weight", "offset": 6790553600, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.16.ffn_down_shexp.weight", "offset": 6791077888, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.16.ffn_gate_shexp.weight", "offset": 6795042816, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.16.ffn_up_shexp.weight", "offset": 6799007744, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 6802972672, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.16.attn_kv_a_norm.weight", "offset": 6802980864, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.16.attn_kv_a_mqa.weight", "offset": 6802982912, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.16.attn_kv_b.weight", "offset": 6803793920, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 6805235712, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.16.attn_q.weight", "offset": 6808119296, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 6812444672, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.17.ffn_down_exps.weight", "offset": 6812452864, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.17.ffn_gate_exps.weight", "offset": 6939330560, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.17.ffn_up_exps.weight", "offset": 7066208256, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.17.ffn_gate_inp.weight", "offset": 7193085952, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.17.ffn_down_shexp.weight", "offset": 7193610240, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.17.ffn_gate_shexp.weight", "offset": 7197575168, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.17.ffn_up_shexp.weight", "offset": 7201540096, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 7205505024, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.17.attn_kv_a_norm.weight", "offset": 7205513216, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.17.attn_kv_a_mqa.weight", "offset": 7205515264, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.17.attn_kv_b.weight", "offset": 7206326272, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 7207768064, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.17.attn_q.weight", "offset": 7210651648, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 7214977024, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.18.ffn_down_exps.weight", "offset": 7214985216, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.18.ffn_gate_exps.weight", "offset": 7341862912, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.18.ffn_up_exps.weight", "offset": 7468740608, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.18.ffn_gate_inp.weight", "offset": 7595618304, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.18.ffn_down_shexp.weight", "offset": 7596142592, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.18.ffn_gate_shexp.weight", "offset": 7600107520, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.18.ffn_up_shexp.weight", "offset": 7604072448, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 7608037376, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.18.attn_kv_a_norm.weight", "offset": 7608045568, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.18.attn_kv_a_mqa.weight", "offset": 7608047616, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.18.attn_kv_b.weight", "offset": 7608858624, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 7610300416, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.18.attn_q.weight", "offset": 7613184000, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 7617509376, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.19.ffn_down_exps.weight", "offset": 7617517568, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.19.ffn_gate_exps.weight", "offset": 7744395264, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.19.ffn_up_exps.weight", "offset": 7871272960, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.19.ffn_gate_inp.weight", "offset": 7998150656, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.19.ffn_down_shexp.weight", "offset": 7998674944, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.19.ffn_gate_shexp.weight", "offset": 8002639872, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.19.ffn_up_shexp.weight", "offset": 8006604800, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 8010569728, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.19.attn_kv_a_norm.weight", "offset": 8010577920, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.19.attn_kv_a_mqa.weight", "offset": 8010579968, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.19.attn_kv_b.weight", "offset": 8011390976, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 8012832768, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.19.attn_q.weight", "offset": 8015716352, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 8020041728, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.20.ffn_down_exps.weight", "offset": 8020049920, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.20.ffn_gate_exps.weight", "offset": 8146927616, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.20.ffn_up_exps.weight", "offset": 8273805312, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.20.ffn_gate_inp.weight", "offset": 8400683008, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.20.ffn_down_shexp.weight", "offset": 8401207296, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.20.ffn_gate_shexp.weight", "offset": 8405172224, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.20.ffn_up_shexp.weight", "offset": 8409137152, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 8413102080, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.20.attn_kv_a_norm.weight", "offset": 8413110272, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.20.attn_kv_a_mqa.weight", "offset": 8413112320, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.20.attn_kv_b.weight", "offset": 8413923328, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 8415365120, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.20.attn_q.weight", "offset": 8418248704, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 8422574080, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.21.ffn_down_exps.weight", "offset": 8422582272, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.21.ffn_gate_exps.weight", "offset": 8549459968, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.21.ffn_up_exps.weight", "offset": 8676337664, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.21.ffn_gate_inp.weight", "offset": 8803215360, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.21.ffn_down_shexp.weight", "offset": 8803739648, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.21.ffn_gate_shexp.weight", "offset": 8807704576, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.21.ffn_up_shexp.weight", "offset": 8811669504, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 8815634432, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.21.attn_kv_a_norm.weight", "offset": 8815642624, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.21.attn_kv_a_mqa.weight", "offset": 8815644672, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.21.attn_kv_b.weight", "offset": 8816455680, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 8817897472, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.21.attn_q.weight", "offset": 8820781056, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.22.ffn_gate_inp.weight", "offset": 8825106432, "shape": [ 2048, 64 ], "size": 4325376, "type": 0 }, { "name": "blk.22.ffn_down_shexp.weight", "offset": 8825630720, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.22.ffn_gate_shexp.weight", "offset": 8829595648, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.22.ffn_up_shexp.weight", "offset": 8833560576, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.22.attn_kv_a_norm.weight", "offset": 8837525504, "shape": [ 512 ], "size": 3964928, "type": 0 }, { "name": "blk.22.attn_kv_a_mqa.weight", "offset": 8837527552, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.22.attn_kv_b.weight", "offset": 8838338560, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 8839780352, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.22.attn_q.weight", "offset": 8842663936, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 8846989312, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.22.ffn_down_exps.weight", "offset": 8846997504, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.22.ffn_gate_exps.weight", "offset": 8973875200, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.22.ffn_up_exps.weight", "offset": 9100752896, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 9227630592, "shape": [ 2048 ], "size": 126877696, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 9227638784, "shape": [ 2048 ], "size": 8192, "type": 0 }, { "name": "blk.23.ffn_down_exps.weight", "offset": 9227646976, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.23.ffn_gate_exps.weight", "offset": 9354524672, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.23.ffn_up_exps.weight", "offset": 9481402368, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.23.ffn_gate_inp.weight", "offset": 9608280064, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.23.ffn_down_shexp.weight", "offset": 9608804352, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.23.ffn_gate_shexp.weight", "offset": 9612769280, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.23.ffn_up_shexp.weight", "offset": 9616734208, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 9620699136, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.23.attn_kv_a_norm.weight", "offset": 9620707328, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.23.attn_kv_a_mqa.weight", "offset": 9620709376, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.23.attn_kv_b.weight", "offset": 9621520384, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 9622962176, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.23.attn_q.weight", "offset": 9625845760, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.24.attn_norm.weight", "offset": 9630171136, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.24.ffn_down_exps.weight", "offset": 9630179328, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.24.ffn_gate_exps.weight", "offset": 9757057024, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.24.ffn_up_exps.weight", "offset": 9883934720, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.24.ffn_gate_inp.weight", "offset": 10010812416, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.24.ffn_down_shexp.weight", "offset": 10011336704, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.24.ffn_gate_shexp.weight", "offset": 10015301632, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.24.ffn_up_shexp.weight", "offset": 10019266560, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.24.ffn_norm.weight", "offset": 10023231488, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.24.attn_kv_a_norm.weight", "offset": 10023239680, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.24.attn_kv_a_mqa.weight", "offset": 10023241728, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.24.attn_kv_b.weight", "offset": 10024052736, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.24.attn_output.weight", "offset": 10025494528, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.24.attn_q.weight", "offset": 10028378112, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.25.attn_norm.weight", "offset": 10032703488, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.25.ffn_down_exps.weight", "offset": 10032711680, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.25.ffn_gate_exps.weight", "offset": 10159589376, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.25.ffn_up_exps.weight", "offset": 10286467072, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.25.ffn_gate_inp.weight", "offset": 10413344768, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.25.ffn_down_shexp.weight", "offset": 10413869056, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.25.ffn_gate_shexp.weight", "offset": 10417833984, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.25.ffn_up_shexp.weight", "offset": 10421798912, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.25.ffn_norm.weight", "offset": 10425763840, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.25.attn_kv_a_norm.weight", "offset": 10425772032, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.25.attn_kv_a_mqa.weight", "offset": 10425774080, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.25.attn_kv_b.weight", "offset": 10426585088, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.25.attn_output.weight", "offset": 10428026880, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.25.attn_q.weight", "offset": 10430910464, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 }, { "name": "blk.26.attn_norm.weight", "offset": 10435235840, "shape": [ 2048 ], "size": 4325376, "type": 0 }, { "name": "blk.26.ffn_down_exps.weight", "offset": 10435244032, "shape": [ 1408, 2048, 64 ], "size": 8192, "type": 6 }, { "name": "blk.26.ffn_gate_exps.weight", "offset": 10562121728, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.26.ffn_up_exps.weight", "offset": 10688999424, "shape": [ 2048, 1408, 64 ], "size": 126877696, "type": 6 }, { "name": "blk.26.ffn_gate_inp.weight", "offset": 10815877120, "shape": [ 2048, 64 ], "size": 126877696, "type": 0 }, { "name": "blk.26.ffn_down_shexp.weight", "offset": 10816401408, "shape": [ 2816, 2048 ], "size": 524288, "type": 6 }, { "name": "blk.26.ffn_gate_shexp.weight", "offset": 10820366336, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.26.ffn_up_shexp.weight", "offset": 10824331264, "shape": [ 2048, 2816 ], "size": 3964928, "type": 6 }, { "name": "blk.26.ffn_norm.weight", "offset": 10828296192, "shape": [ 2048 ], "size": 3964928, "type": 0 }, { "name": "blk.26.attn_kv_a_norm.weight", "offset": 10828304384, "shape": [ 512 ], "size": 8192, "type": 0 }, { "name": "blk.26.attn_kv_a_mqa.weight", "offset": 10828306432, "shape": [ 2048, 576 ], "size": 2048, "type": 6 }, { "name": "blk.26.attn_kv_b.weight", "offset": 10829117440, "shape": [ 512, 4096 ], "size": 811008, "type": 6 }, { "name": "blk.26.attn_output.weight", "offset": 10830559232, "shape": [ 2048, 2048 ], "size": 1441792, "type": 6 }, { "name": "blk.26.attn_q.weight", "offset": 10833442816, "shape": [ 2048, 3072 ], "size": 2883584, "type": 6 } ], "version": 3 }