gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 18, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 42162176, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 84310016, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.0.post_attention_norm.weight", "offset": 126457856, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 126472192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 126486528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 126500864, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 132521984, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.0.attn_q.weight", "offset": 144564224, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.0.attn_v.weight", "offset": 156606464, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 162627584, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 162641920, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 204789760, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 246937600, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.1.post_attention_norm.weight", "offset": 289085440, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 289099776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 289114112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 289128448, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 295149568, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.1.attn_q.weight", "offset": 307191808, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.1.attn_v.weight", "offset": 319234048, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.2.attn_v.weight", "offset": 325255168, "shape": [ 3584, 2048 ], "size": 6021120, "type": 14 }, { "name": "blk.2.ffn_down.weight", "offset": 331276288, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 373424128, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 415571968, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.2.post_attention_norm.weight", "offset": 457719808, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 457734144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 457748480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 457762816, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 463783936, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.2.attn_q.weight", "offset": 475826176, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 487868416, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 487882752, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.3.ffn_down.weight", "offset": 493903872, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 536051712, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 578199552, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.3.post_attention_norm.weight", "offset": 620347392, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 620361728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 620376064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 620390400, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 626411520, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.3.attn_q.weight", "offset": 638453760, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 650496000, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 650510336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 650524672, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 692672512, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 734820352, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.4.post_attention_norm.weight", "offset": 776968192, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 776982528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 776996864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 777011200, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 783032320, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.4.attn_q.weight", "offset": 795074560, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.4.attn_v.weight", "offset": 807116800, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 813137920, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 813152256, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 855300096, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 897447936, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.17.attn_v.weight\u0002\u0000\u0000\u0000\u0000\u000e", "offset": 939595776, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 939610112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 939624448, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 939638784, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 945659904, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.5.attn_q.weight", "offset": 957702144, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.5.attn_v.weight", "offset": 969744384, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 975765504, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 975779840, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 1017927680, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 1060075520, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.6.post_attention_norm.weight", "offset": 1102223360, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 1102237696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 1102252032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1102266368, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 1108287488, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.6.attn_q.weight", "offset": 1120329728, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.6.attn_v.weight", "offset": 1132371968, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 1138393088, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 1138407424, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.7.attn_k.weight", "offset": 1180555264, "shape": [ 3584, 2048 ], "size": 42147840, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 1186576384, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.7.attn_q.weight", "offset": 1198618624, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.7.attn_v.weight", "offset": 1210660864, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 1216681984, "shape": [ 3584, 14336 ], "size": 6021120, "type": 14 }, { "name": "blk.7.ffn_down.weight", "offset": 1258829824, "shape": [ 14336, 3584 ], "size": 42147840, "type": 14 }, { "name": "blk.7.post_attention_norm.weight", "offset": 1300977664, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 1300992000, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1301006336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 1301020672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1301035008, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 1343182848, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 1385330688, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1427478528, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1427492864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1427507200, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 1433528320, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.8.attn_q.weight", "offset": 1445570560, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.8.attn_v.weight", "offset": 1457612800, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1463633920, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1463648256, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.9.ffn_down.weight", "offset": 1469669376, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 1511817216, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1511831552, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 1553979392, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1596127232, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1596141568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1596155904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1596170240, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 1602191360, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.9.attn_q.weight", "offset": 1614233600, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.10.attn_q.weight", "offset": 1626275840, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.10.attn_v.weight", "offset": 1638318080, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.10.ffn_down.weight", "offset": 1644339200, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 1686487040, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.10.attn_k.weight", "offset": 1728634880, "shape": [ 3584, 2048 ], "size": 42147840, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 1734656000, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1734670336, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1776818176, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1776832512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1776846848, "shape": [ 4096, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 1788889088, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1788903424, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1788917760, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1788932096, "shape": [ 3584, 4096 ], "size": 14336, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 1800974336, "shape": [ 3584, 14336 ], "size": 12042240, "type": 14 }, { "name": "blk.11.attn_k.weight", "offset": 1843122176, "shape": [ 3584, 2048 ], "size": 42147840, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 1849143296, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1849157632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1849171968, "shape": [ 4096, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.11.attn_v.weight", "offset": 1861214208, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.11.ffn_down.weight", "offset": 1867235328, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 1909383168, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 1951531008, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1951545344, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.12.attn_v.weight", "offset": 1993693184, "shape": [ 3584, 2048 ], "size": 42147840, "type": 14 }, { "name": "blk.12.attn_q.weight", "offset": 1999714304, "shape": [ 3584, 4096 ], "size": 6021120, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 2011756544, "shape": [ 4096, 3584 ], "size": 12042240, "type": 14 }, { "name": "blk.12.attn_k.weight", "offset": 2023798784, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 2029819904, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 2029834240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 2029848576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 2029862912, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 2072010752, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 2114158592, "shape": [ 4096, 3584 ], "size": 42147840, "type": 14 }, { "name": "blk.13.attn_v.weight", "offset": 2126200832, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 2132221952, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 2132236288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 2132250624, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 2132264960, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 2174412800, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.13.ffn_down.weight", "offset": 2216560640, "shape": [ 14336, 3584 ], "size": 42147840, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 2258708480, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2258722816, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.13.attn_q.weight", "offset": 2264743936, "shape": [ 3584, 4096 ], "size": 6021120, "type": 14 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 2276786176, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 2276800512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2276814848, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 2318962688, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 2361110528, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.14.post_attention_norm.weight", "offset": 2403258368, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2403272704, "shape": [ 4096, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 2415314944, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2415329280, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.14.attn_v.weight", "offset": 2421350400, "shape": [ 3584, 2048 ], "size": 6021120, "type": 14 }, { "name": "blk.14.attn_q.weight", "offset": 2427371520, "shape": [ 3584, 4096 ], "size": 6021120, "type": 14 }, { "name": "blk.15.attn_v.weight", "offset": 2439413760, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.15.attn_q.weight", "offset": 2445434880, "shape": [ 3584, 4096 ], "size": 6021120, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 2457477120, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2457491456, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 2499639296, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 2541787136, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.15.post_attention_norm.weight", "offset": 2583934976, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 2583949312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 2583963648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2583977984, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 2589999104, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 2602041344, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 2602055680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2602070016, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 2608091136, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2608105472, "shape": [ 3584, 4096 ], "size": 14336, "type": 14 }, { "name": "blk.16.attn_v.weight", "offset": 2620147712, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 2626168832, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.16.post_attention_norm.weight", "offset": 2638211072, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 2638225408, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 2680373248, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.16.ffn_down.weight", "offset": 2722521088, "shape": [ 14336, 3584 ], "size": 42147840, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 2764668928, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2764683264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2764697600, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 2770718720, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 2782760960, "shape": [ 3584 ], "size": 12042240, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 2782775296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 2782789632, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 2824937472, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.17.ffn_down.weight", "offset": 2867085312, "shape": [ 14336, 3584 ], "size": 42147840, "type": 14 }, { "name": "ffn_gate.weight\u0002\u0000\u0000\u0000\u0000", "offset": 2909233152, "shape": [ 3584, 2048 ], "size": 42147840, "type": 14 }, { "name": "blk.17.attn_q.weight", "offset": 2915254272, "shape": [ 3584, 4096 ], "size": 6021120, "type": 14 }, { "name": "blk.18.attn_v.weight", "offset": 2927296512, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2933317632, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2933331968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2933346304, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 2975494144, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 3017641984, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.18.post_attention_norm.weight", "offset": 3059789824, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 3059804160, "shape": [ 3584, 4096 ], "size": 14336, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 3071846400, "shape": [ 4096, 3584 ], "size": 12042240, "type": 14 }, { "name": "blk.18.attn_k.weight", "offset": 3083888640, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 3089909760, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 3089924096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 3089938432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3089952768, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 3132100608, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 3174248448, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.19.post_attention_norm.weight", "offset": 3216396288, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 3216410624, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3216424960, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 3222446080, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.19.attn_q.weight", "offset": 3234488320, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.19.attn_v.weight", "offset": 3246530560, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.20.attn_k.weight", "offset": 3252551680, "shape": [ 3584, 2048 ], "size": 6021120, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 3258572800, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.20.attn_q.weight", "offset": 3270615040, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.20.attn_v.weight", "offset": 3282657280, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 3288678400, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3288692736, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 3330840576, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 3372988416, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.20.post_attention_norm.weight", "offset": 3415136256, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 3415150592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 3415164928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3415179264, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 3457327104, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 3457341440, "shape": [ 3584, 14336 ], "size": 14336, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 3499489280, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.21.post_attention_norm.weight", "offset": 3541637120, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 3541651456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 3541665792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3541680128, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 3547701248, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.21.attn_q.weight", "offset": 3559743488, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.21.attn_v.weight", "offset": 3571785728, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 3577806848, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3577821184, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 3619969024, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 3662116864, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.22.post_attention_norm.weight", "offset": 3704264704, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 3704279040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 3704293376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3704307712, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 3710328832, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.22.attn_q.weight", "offset": 3722371072, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.22.attn_v.weight", "offset": 3734413312, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 3740434432, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3740448768, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 3782596608, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 3824744448, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.23.post_attention_norm.weight", "offset": 3866892288, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 3866906624, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 3866920960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3866935296, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 3872956416, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.23.attn_q.weight", "offset": 3884998656, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.23.attn_v.weight", "offset": 3897040896, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 3903062016, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3903076352, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 3945224192, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 3987372032, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.24.post_attention_norm.weight", "offset": 4029519872, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 4029534208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 4029548544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4029562880, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 4035584000, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.24.attn_q.weight", "offset": 4047626240, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.24.attn_v.weight", "offset": 4059668480, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 4065689600, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4065703936, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 4107851776, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 4149999616, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.25.post_attention_norm.weight", "offset": 4192147456, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 4192161792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 4192176128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4192190464, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 4198211584, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.25.attn_q.weight", "offset": 4210253824, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.25.attn_v.weight", "offset": 4222296064, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 4228317184, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4228331520, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 4270479360, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 4312627200, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.26.post_attention_norm.weight", "offset": 4354775040, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 4354789376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 4354803712, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4354818048, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 4360839168, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.26.attn_q.weight", "offset": 4372881408, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.26.attn_v.weight", "offset": 4384923648, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 4390944768, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4390959104, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 4433106944, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 4475254784, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.27.post_attention_norm.weight", "offset": 4517402624, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 4517416960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 4517431296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4517445632, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 4523466752, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.27.attn_q.weight", "offset": 4535508992, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.27.attn_v.weight", "offset": 4547551232, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 4553572352, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4553586688, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 4595734528, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 4637882368, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.28.post_attention_norm.weight", "offset": 4680030208, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 4680044544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 4680058880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 4680073216, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 4686094336, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.28.attn_q.weight", "offset": 4698136576, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.28.attn_v.weight", "offset": 4710178816, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 4716199936, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4716214272, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.29.ffn_gate.weight", "offset": 4758362112, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 4800509952, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.29.post_attention_norm.weight", "offset": 4842657792, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 4842672128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 4842686464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 4842700800, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.29.attn_output.weight", "offset": 4848721920, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.29.attn_q.weight", "offset": 4860764160, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.29.attn_v.weight", "offset": 4872806400, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 4878827520, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4878841856, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "\ufffd?\u0000\ufffd\ufffd?\u0000@\ufffd?\u0000\ufffd\ufffd?\u0000\u0000\ufffd?\u0000\ufffd\ufffd?", "offset": 4920989696, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 4963137536, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.30.post_attention_norm.weight", "offset": 5005285376, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 5005299712, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 5005314048, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 5005328384, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.30.attn_output.weight", "offset": 5011349504, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.30.attn_q.weight", "offset": 5023391744, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.30.attn_v.weight", "offset": 5035433984, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.31.attn_k.weight", "offset": 5041455104, "shape": [ 3584, 2048 ], "size": 6021120, "type": 14 }, { "name": "blk.31.ffn_down.weight", "offset": 5047476224, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 5089624064, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 5131771904, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.31.post_attention_norm.weight", "offset": 5173919744, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 5173934080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 5173948416, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 5173962752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 5173977088, "shape": [ 4096, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.31.attn_q.weight", "offset": 5186019328, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.31.attn_v.weight", "offset": 5198061568, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 5204082688, "shape": [ 3584, 14336 ], "size": 6021120, "type": 14 }, { "name": "blk.32.ffn_up.weight", "offset": 5246230528, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.32.attn_k.weight", "offset": 5288378368, "shape": [ 3584, 2048 ], "size": 42147840, "type": 14 }, { "name": "blk.32.attn_output.weight", "offset": 5294399488, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.32.attn_q.weight", "offset": 5306441728, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.32.attn_v.weight", "offset": 5318483968, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 5324505088, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 5324519424, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.32.post_attention_norm.weight", "offset": 5366667264, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 5366681600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 5366695936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 5366710272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 5366724608, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.33.ffn_gate.weight", "offset": 5408872448, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.33.ffn_up.weight", "offset": 5451020288, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.33.post_attention_norm.weight", "offset": 5493168128, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 5493182464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 5493196800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 5493211136, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.33.attn_output.weight", "offset": 5499232256, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.33.attn_q.weight", "offset": 5511274496, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.33.attn_v.weight", "offset": 5523316736, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 5529337856, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 5529352192, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 5571500032, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 5613647872, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.34.post_attention_norm.weight", "offset": 5655795712, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 5655810048, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 5655824384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 5655838720, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.34.attn_output.weight", "offset": 5661859840, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.34.attn_q.weight", "offset": 5673902080, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.34.attn_v.weight", "offset": 5685944320, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 5691965440, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 5691979776, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.35.ffn_gate.weight", "offset": 5734127616, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 5776275456, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.35.post_attention_norm.weight", "offset": 5818423296, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 5818437632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 5818451968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 5818466304, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 5824487424, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.35.attn_q.weight", "offset": 5836529664, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.35.attn_v.weight", "offset": 5848571904, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.36.ffn_norm.weight", "offset": 5854593024, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 5854607360, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 5896755200, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 5938903040, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.36.post_attention_norm.weight", "offset": 5981050880, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 5981065216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 5981079552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 5981093888, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.36.attn_output.weight", "offset": 5987115008, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.36.attn_q.weight", "offset": 5999157248, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.36.attn_v.weight", "offset": 6011199488, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 6017220608, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 6017234944, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 6059382784, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 6101530624, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.37.post_attention_norm.weight", "offset": 6143678464, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 6143692800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 6143707136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 6143721472, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.37.attn_output.weight", "offset": 6149742592, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.37.attn_q.weight", "offset": 6161784832, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.37.attn_v.weight", "offset": 6173827072, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 6179848192, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 6179862528, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.38.ffn_gate.weight", "offset": 6222010368, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 6264158208, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.38.post_attention_norm.weight", "offset": 6306306048, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 6306320384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 6306334720, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 6306349056, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.38.attn_output.weight", "offset": 6312370176, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.38.attn_q.weight", "offset": 6324412416, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.38.attn_v.weight", "offset": 6336454656, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 6342475776, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 6342490112, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 6384637952, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 6426785792, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.39.post_attention_norm.weight", "offset": 6468933632, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 6468947968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 6468962304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 6468976640, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.39.attn_output.weight", "offset": 6474997760, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.39.attn_q.weight", "offset": 6487040000, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.39.attn_v.weight", "offset": 6499082240, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 6505103360, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 6505117696, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.40.ffn_gate.weight", "offset": 6547265536, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.40.ffn_up.weight", "offset": 6589413376, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.40.post_attention_norm.weight", "offset": 6631561216, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 6631575552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 6631589888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 6631604224, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.40.attn_output.weight", "offset": 6637625344, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.40.attn_q.weight", "offset": 6649667584, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.40.attn_v.weight", "offset": 6661709824, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 6667730944, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 6667745280, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.41.ffn_gate.weight", "offset": 6709893120, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.41.ffn_up.weight", "offset": 6752040960, "shape": [ 3584, 14336 ], "size": 42147840, "type": 14 }, { "name": "blk.41.post_attention_norm.weight", "offset": 6794188800, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 6794203136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 6794217472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 6794231808, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.41.attn_output.weight", "offset": 6800252928, "shape": [ 4096, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.41.attn_q.weight", "offset": 6812295168, "shape": [ 3584, 4096 ], "size": 12042240, "type": 14 }, { "name": "blk.41.attn_v.weight", "offset": 6824337408, "shape": [ 3584, 2048 ], "size": 12042240, "type": 14 }, { "name": "output_norm.weight", "offset": 6830358528, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "token_embd.weight", "offset": 6830372864, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }