gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 17, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 42162176, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 77486080, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.0.post_attention_norm.weight", "offset": 112809984, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 112824320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 112838656, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 112852992, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 117899264, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 127991808, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.0.attn_v.weight", "offset": 138084352, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 144105472, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 144119808, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 186267648, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 221591552, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.1.post_attention_norm.weight", "offset": 256915456, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 256929792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 256944128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 256958464, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 262004736, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 272097280, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.1.attn_v.weight", "offset": 282189824, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.2.attn_v.weight", "offset": 288210944, "shape": [ 3584, 2048 ], "size": 6021120, "type": 14 }, { "name": "blk.2.ffn_down.weight", "offset": 294232064, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 336379904, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 371703808, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.2.post_attention_norm.weight", "offset": 407027712, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 407042048, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 407056384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 407070720, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 412116992, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 422209536, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 432302080, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 432316416, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.3.ffn_down.weight", "offset": 438337536, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 480485376, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 515809280, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.3.post_attention_norm.weight", "offset": 551133184, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 551147520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 551161856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 551176192, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 556222464, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 566315008, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 576407552, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 576421888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 576436224, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 618584064, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 653907968, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.4.post_attention_norm.weight", "offset": 689231872, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 689246208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 689260544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 689274880, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 694321152, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 704413696, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.4.attn_v.weight", "offset": 714506240, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 720527360, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 720541696, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 755865600, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 791189504, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.5.post_attention_norm.weight", "offset": 826513408, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 826527744, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 826542080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 826556416, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 831602688, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 841695232, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.5.attn_v.weight", "offset": 851787776, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 856834048, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 856848384, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 892172288, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 927496192, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.6.post_attention_norm.weight", "offset": 962820096, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 962834432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 962848768, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 962863104, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 967909376, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 978001920, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.6.attn_v.weight", "offset": 988094464, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 993140736, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 993155072, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.7.attn_k.weight", "offset": 1028478976, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 1033525248, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 1043617792, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.7.attn_v.weight", "offset": 1053710336, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 1059731456, "shape": [ 3584, 14336 ], "size": 6021120, "type": 13 }, { "name": "blk.7.ffn_down.weight", "offset": 1095055360, "shape": [ 14336, 3584 ], "size": 35323904, "type": 14 }, { "name": "blk.7.post_attention_norm.weight", "offset": 1137203200, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 1137217536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1137231872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 1137246208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1137260544, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 1172584448, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 1207908352, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1243232256, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1243246592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1243260928, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 1248307200, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 1258399744, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.8.attn_v.weight", "offset": 1268492288, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1273538560, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1273552896, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.9.ffn_down.weight", "offset": 1278599168, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1313923072, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1313937408, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 1349261312, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1384585216, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1384599552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1384613888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1384628224, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 1389674496, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 1399767040, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 1409859584, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.10.attn_v.weight", "offset": 1419952128, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.10.ffn_down.weight", "offset": 1425973248, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 1468121088, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.10.attn_k.weight", "offset": 1503444992, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 1508491264, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1508505600, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1543829504, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1543843840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1543858176, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1553950720, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1553965056, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1553979392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1553993728, "shape": [ 3584, 4096 ], "size": 14336, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 1564086272, "shape": [ 3584, 14336 ], "size": 10092544, "type": 13 }, { "name": "blk.11.attn_k.weight", "offset": 1599410176, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1604456448, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1604470784, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1604485120, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 1614577664, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.11.ffn_down.weight", "offset": 1619623936, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1654947840, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1690271744, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1690286080, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 1725609984, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 1730656256, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 1740748800, "shape": [ 4096, 3584 ], "size": 10092544, "type": 13 }, { "name": "blk.12.attn_k.weight", "offset": 1750841344, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 1755887616, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1755901952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1755916288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1755930624, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 1791254528, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 1826578432, "shape": [ 4096, 3584 ], "size": 35323904, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 1836670976, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 1842692096, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1842706432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1842720768, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1842735104, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1878059008, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.13.ffn_down.weight", "offset": 1913382912, "shape": [ 14336, 3584 ], "size": 35323904, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 1955530752, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1955545088, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 1960591360, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1970683904, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1970698240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1970712576, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 2006036480, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 2041360384, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.14.post_attention_norm.weight", "offset": 2076684288, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2076698624, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 2086791168, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2086805504, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.14.attn_v.weight", "offset": 2091851776, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 2096898048, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.15.attn_v.weight", "offset": 2106990592, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 2112036864, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 2122129408, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2122143744, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 2157467648, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 2192791552, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.15.post_attention_norm.weight", "offset": 2228115456, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 2228129792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 2228144128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2228158464, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 2233204736, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 2243297280, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 2243311616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2243325952, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 2248372224, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2248386560, "shape": [ 3584, 4096 ], "size": 14336, "type": 13 }, { "name": "blk.16.attn_v.weight", "offset": 2258479104, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 2264500224, "shape": [ 4096, 3584 ], "size": 6021120, "type": 13 }, { "name": "blk.16.post_attention_norm.weight", "offset": 2274592768, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 2274607104, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 2309931008, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.16.ffn_down.weight", "offset": 2345254912, "shape": [ 14336, 3584 ], "size": 35323904, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 2387402752, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2387417088, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2387431424, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 2392477696, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 2402570240, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 2402584576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 2402598912, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 2437922816, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.17.ffn_down.weight", "offset": 2473246720, "shape": [ 14336, 3584 ], "size": 35323904, "type": 13 }, { "name": "blk.17.attn_v.weight", "offset": 2508570624, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 2513616896, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.18.attn_v.weight", "offset": 2523709440, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2528755712, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2528770048, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2528784384, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 2564108288, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 2599432192, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.18.post_attention_norm.weight", "offset": 2634756096, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2634770432, "shape": [ 3584, 4096 ], "size": 14336, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 2644862976, "shape": [ 4096, 3584 ], "size": 10092544, "type": 13 }, { "name": "blk.18.attn_k.weight", "offset": 2654955520, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 2660001792, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2660016128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2660030464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2660044800, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 2702192640, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 2737516544, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2772840448, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2772854784, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2772869120, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 2777915392, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 2788007936, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.19.attn_v.weight", "offset": 2798100480, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.20.attn_k.weight", "offset": 2804121600, "shape": [ 3584, 2048 ], "size": 6021120, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 2809167872, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 2819260416, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.20.attn_v.weight", "offset": 2829352960, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 2834399232, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2834413568, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 2869737472, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 2905061376, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2940385280, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2940399616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2940413952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2940428288, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2975752192, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2975766528, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 3011090432, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.21.post_attention_norm.weight", "offset": 3046414336, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 3046428672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 3046443008, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3046457344, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 3051503616, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 3061596160, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.21.attn_v.weight", "offset": 3071688704, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 3076734976, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3076749312, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 3118897152, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 3154221056, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.22.post_attention_norm.weight", "offset": 3189544960, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 3189559296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 3189573632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3189587968, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 3194634240, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 3204726784, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.22.attn_v.weight", "offset": 3214819328, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 3220840448, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3220854784, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 3256178688, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 3291502592, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.23.post_attention_norm.weight", "offset": 3326826496, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 3326840832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 3326855168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3326869504, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 3331915776, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 3342008320, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 3352100864, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 3357147136, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3357161472, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 3392485376, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 3427809280, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.24.post_attention_norm.weight", "offset": 3463133184, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 3463147520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 3463161856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3463176192, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 3468222464, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 3478315008, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 3488407552, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 3493453824, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3493468160, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 3535616000, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 3570939904, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.25.post_attention_norm.weight", "offset": 3606263808, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 3606278144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 3606292480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3606306816, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 3611353088, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 3621445632, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.25.attn_v.weight", "offset": 3631538176, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 3637559296, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3637573632, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 3672897536, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 3708221440, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.26.post_attention_norm.weight", "offset": 3743545344, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 3743559680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3743574016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3743588352, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 3748634624, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 3758727168, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.26.attn_v.weight", "offset": 3768819712, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 3773865984, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3773880320, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 3809204224, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 3844528128, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.27.post_attention_norm.weight", "offset": 3879852032, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 3879866368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3879880704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3879895040, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 3884941312, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 3895033856, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.27.attn_v.weight", "offset": 3905126400, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 3910172672, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3910187008, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 3952334848, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 3987658752, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.28.post_attention_norm.weight", "offset": 4022982656, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 4022996992, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 4023011328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 4023025664, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 4028071936, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 4038164480, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.28.attn_v.weight", "offset": 4048257024, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 4054278144, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4054292480, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 4089616384, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 4124940288, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.29.post_attention_norm.weight", "offset": 4160264192, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 4160278528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 4160292864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 4160307200, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 4165353472, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 4175446016, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.29.attn_v.weight", "offset": 4185538560, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 4190584832, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4190599168, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 4225923072, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 4261246976, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.30.post_attention_norm.weight", "offset": 4296570880, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 4296585216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 4296599552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4296613888, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 4301660160, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 4311752704, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.30.attn_v.weight", "offset": 4321845248, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.31.attn_k.weight", "offset": 4326891520, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.31.ffn_down.weight", "offset": 4331937792, "shape": [ 14336, 3584 ], "size": 5046272, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 4374085632, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 4409409536, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.31.post_attention_norm.weight", "offset": 4444733440, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 4444747776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 4444762112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 4444776448, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 4444790784, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 4454883328, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.31.attn_v.weight", "offset": 4464975872, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 4470996992, "shape": [ 3584, 14336 ], "size": 6021120, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 4506320896, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.32.attn_k.weight", "offset": 4541644800, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 4546691072, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 4556783616, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.32.attn_v.weight", "offset": 4566876160, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 4571922432, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4571936768, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.32.post_attention_norm.weight", "offset": 4607260672, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 4607275008, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 4607289344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 4607303680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4607318016, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 4642641920, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 4677965824, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.33.post_attention_norm.weight", "offset": 4713289728, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 4713304064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 4713318400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 4713332736, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 4718379008, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 4728471552, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.33.attn_v.weight", "offset": 4738564096, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 4743610368, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4743624704, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 4785772544, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.34.ffn_up.weight", "offset": 4821096448, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.34.post_attention_norm.weight", "offset": 4856420352, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 4856434688, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 4856449024, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 4856463360, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 4861509632, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 4871602176, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.34.attn_v.weight", "offset": 4881694720, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 4887715840, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4887730176, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 4923054080, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 4958377984, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.35.post_attention_norm.weight", "offset": 4993701888, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 4993716224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 4993730560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4993744896, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 4998791168, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 5008883712, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.35.attn_v.weight", "offset": 5018976256, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 5024022528, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 5024036864, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 5066184704, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 5101508608, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.36.post_attention_norm.weight", "offset": 5136832512, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 5136846848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 5136861184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 5136875520, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 5141921792, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 5152014336, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.36.attn_v.weight", "offset": 5162106880, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 5168128000, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 5168142336, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 5210290176, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 5245614080, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.37.post_attention_norm.weight", "offset": 5280937984, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 5280952320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 5280966656, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 5280980992, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 5286027264, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 5296119808, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.37.attn_v.weight", "offset": 5306212352, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 5312233472, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5312247808, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.38.ffn_gate.weight", "offset": 5354395648, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 5389719552, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.38.post_attention_norm.weight", "offset": 5425043456, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 5425057792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 5425072128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 5425086464, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 5430132736, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 5440225280, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.38.attn_v.weight", "offset": 5450317824, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 5456338944, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5456353280, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 5498501120, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 5533825024, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.39.post_attention_norm.weight", "offset": 5569148928, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 5569163264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 5569177600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 5569191936, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 5574238208, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 5584330752, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.39.attn_v.weight", "offset": 5594423296, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 5600444416, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 5600458752, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.40.ffn_gate.weight", "offset": 5642606592, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.40.ffn_up.weight", "offset": 5677930496, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.40.post_attention_norm.weight", "offset": 5713254400, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 5713268736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 5713283072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 5713297408, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 5718343680, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 5728436224, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.40.attn_v.weight", "offset": 5738528768, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 5744549888, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 5744564224, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.41.ffn_gate.weight", "offset": 5786712064, "shape": [ 3584, 14336 ], "size": 42147840, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 5822035968, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.41.post_attention_norm.weight", "offset": 5857359872, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 5857374208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 5857388544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 5857402880, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 5862449152, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 5872541696, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.41.attn_v.weight", "offset": 5882634240, "shape": [ 3584, 2048 ], "size": 10092544, "type": 14 }, { "name": "output_norm.weight", "offset": 5888655360, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "token_embd.weight", "offset": 5888669696, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }