gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 16, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 35338240, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 70662144, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.0.post_attention_norm.weight", "offset": 105986048, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 106000384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 106014720, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 106029056, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 111075328, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 121167872, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.0.attn_v.weight", "offset": 131260416, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 136306688, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 136321024, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 171644928, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 206968832, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.1.post_attention_norm.weight", "offset": 242292736, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 242307072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 242321408, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 242335744, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 247382016, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 257474560, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.1.attn_v.weight", "offset": 267567104, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 272613376, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.2.ffn_down.weight", "offset": 277659648, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 312983552, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 348307456, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.2.post_attention_norm.weight", "offset": 383631360, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 383645696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 383660032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 383674368, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 388720640, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 398813184, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 408905728, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 408920064, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.3.ffn_down.weight", "offset": 413966336, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 449290240, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 484614144, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.3.post_attention_norm.weight", "offset": 519938048, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 519952384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 519966720, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 519981056, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 525027328, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 535119872, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 545212416, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 545226752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 545241088, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 580564992, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 615888896, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.4.post_attention_norm.weight", "offset": 651212800, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 651227136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 651241472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 651255808, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 656302080, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 666394624, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.4.attn_v.weight", "offset": 676487168, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 681533440, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 681547776, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 716871680, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 752195584, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.5.post_attention_norm.weight", "offset": 787519488, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 787533824, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 787548160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 787562496, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 792608768, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 802701312, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.5.attn_v.weight", "offset": 812793856, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 817840128, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 817854464, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 853178368, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 888502272, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.6.post_attention_norm.weight", "offset": 923826176, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 923840512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 923854848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 923869184, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 928915456, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 939008000, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.6.attn_v.weight", "offset": 949100544, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 954146816, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 954161152, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.7.attn_k.weight", "offset": 989485056, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 994531328, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 1004623872, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.7.attn_v.weight", "offset": 1014716416, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 1019762688, "shape": [ 3584, 14336 ], "size": 5046272, "type": 13 }, { "name": "blk.7.ffn_down.weight", "offset": 1055086592, "shape": [ 14336, 3584 ], "size": 35323904, "type": 13 }, { "name": "blk.7.post_attention_norm.weight", "offset": 1090410496, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 1090424832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 1090439168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 1090453504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1090467840, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 1125791744, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 1161115648, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1196439552, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1196453888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1196468224, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 1201514496, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 1211607040, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.8.attn_v.weight", "offset": 1221699584, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1226745856, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1226760192, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.9.ffn_down.weight", "offset": 1231806464, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1267130368, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1267144704, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 1302468608, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1337792512, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1337806848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1337821184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1337835520, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 1342881792, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 1352974336, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.10.attn_q.weight", "offset": 1363066880, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.10.attn_v.weight", "offset": 1373159424, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.10.ffn_down.weight", "offset": 1378205696, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 1413529600, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.10.attn_k.weight", "offset": 1448853504, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 1453899776, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1453914112, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1489238016, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1489252352, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1489266688, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1499359232, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1499373568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1499387904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1499402240, "shape": [ 3584, 4096 ], "size": 14336, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 1509494784, "shape": [ 3584, 14336 ], "size": 10092544, "type": 13 }, { "name": "blk.11.attn_k.weight", "offset": 1544818688, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1549864960, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1549879296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1549893632, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 1559986176, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.11.ffn_down.weight", "offset": 1565032448, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1600356352, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1635680256, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1635694592, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 1671018496, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 1676064768, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 1686157312, "shape": [ 4096, 3584 ], "size": 10092544, "type": 13 }, { "name": "blk.12.attn_k.weight", "offset": 1696249856, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 1701296128, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1701310464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1701324800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1701339136, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 1736663040, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 1771986944, "shape": [ 4096, 3584 ], "size": 35323904, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 1782079488, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 1787125760, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1787140096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1787154432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1787168768, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1822492672, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.13.ffn_down.weight", "offset": 1857816576, "shape": [ 14336, 3584 ], "size": 35323904, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 1893140480, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1893154816, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.13.attn_q.weight", "offset": 1898201088, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1908293632, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1908307968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1908322304, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 1943646208, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 1978970112, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.14.post_attention_norm.weight", "offset": 2014294016, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2014308352, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 2024400896, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2024415232, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.14.attn_v.weight", "offset": 2029461504, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 2034507776, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.15.attn_v.weight", "offset": 2044600320, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 2049646592, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 2059739136, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2059753472, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 2095077376, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 2130401280, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.15.post_attention_norm.weight", "offset": 2165725184, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 2165739520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 2165753856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2165768192, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 2170814464, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 2180907008, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 2180921344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2180935680, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 2185981952, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2185996288, "shape": [ 3584, 4096 ], "size": 14336, "type": 13 }, { "name": "blk.16.attn_v.weight", "offset": 2196088832, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 2201135104, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.16.post_attention_norm.weight", "offset": 2211227648, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 2211241984, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 2246565888, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.16.ffn_down.weight", "offset": 2281889792, "shape": [ 14336, 3584 ], "size": 35323904, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 2317213696, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2317228032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2317242368, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 2322288640, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 2332381184, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 2332395520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 2332409856, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 2367733760, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.17.ffn_down.weight", "offset": 2403057664, "shape": [ 14336, 3584 ], "size": 35323904, "type": 13 }, { "name": "blk.17.attn_v.weight", "offset": 2438381568, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 2443427840, "shape": [ 3584, 4096 ], "size": 5046272, "type": 13 }, { "name": "blk.18.attn_v.weight", "offset": 2453520384, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2458566656, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2458580992, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2458595328, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 2493919232, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 2529243136, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.18.post_attention_norm.weight", "offset": 2564567040, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2564581376, "shape": [ 3584, 4096 ], "size": 14336, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 2574673920, "shape": [ 4096, 3584 ], "size": 10092544, "type": 13 }, { "name": "blk.18.attn_k.weight", "offset": 2584766464, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 2589812736, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2589827072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2589841408, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2589855744, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 2625179648, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 2660503552, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2695827456, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2695841792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2695856128, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 2700902400, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 2710994944, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.19.attn_v.weight", "offset": 2721087488, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.20.attn_k.weight", "offset": 2726133760, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 2731180032, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 2741272576, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.20.attn_v.weight", "offset": 2751365120, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 2756411392, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2756425728, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 2791749632, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 2827073536, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2862397440, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2862411776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2862426112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2862440448, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2897764352, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2897778688, "shape": [ 3584, 14336 ], "size": 14336, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 2933102592, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.21.post_attention_norm.weight", "offset": 2968426496, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 2968440832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2968455168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2968469504, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 2973515776, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 2983608320, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.21.attn_v.weight", "offset": 2993700864, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 2998747136, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2998761472, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 3034085376, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 3069409280, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.22.post_attention_norm.weight", "offset": 3104733184, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 3104747520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 3104761856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3104776192, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 3109822464, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 3119915008, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.22.attn_v.weight", "offset": 3130007552, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 3135053824, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3135068160, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 3170392064, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 3205715968, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.23.post_attention_norm.weight", "offset": 3241039872, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 3241054208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 3241068544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3241082880, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 3246129152, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 3256221696, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.23.attn_v.weight", "offset": 3266314240, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 3271360512, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3271374848, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 3306698752, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 3342022656, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.24.post_attention_norm.weight", "offset": 3377346560, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 3377360896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 3377375232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3377389568, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 3382435840, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 3392528384, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.24.attn_v.weight", "offset": 3402620928, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 3407667200, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3407681536, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 3443005440, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 3478329344, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.25.post_attention_norm.weight", "offset": 3513653248, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 3513667584, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 3513681920, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3513696256, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 3518742528, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 3528835072, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.25.attn_v.weight", "offset": 3538927616, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 3543973888, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3543988224, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 3579312128, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 3614636032, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.26.post_attention_norm.weight", "offset": 3649959936, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 3649974272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3649988608, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3650002944, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 3655049216, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 3665141760, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.26.attn_v.weight", "offset": 3675234304, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 3680280576, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3680294912, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 3715618816, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 3750942720, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.27.post_attention_norm.weight", "offset": 3786266624, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 3786280960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3786295296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3786309632, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 3791355904, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 3801448448, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.27.attn_v.weight", "offset": 3811540992, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 3816587264, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3816601600, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 3851925504, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 3887249408, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.28.post_attention_norm.weight", "offset": 3922573312, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 3922587648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3922601984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3922616320, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 3927662592, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 3937755136, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.28.attn_v.weight", "offset": 3947847680, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 3952893952, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3952908288, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 3988232192, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 4023556096, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.29.post_attention_norm.weight", "offset": 4058880000, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 4058894336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 4058908672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 4058923008, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.29.attn_output.weight", "offset": 4063969280, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 4074061824, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.29.attn_v.weight", "offset": 4084154368, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 4089200640, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4089214976, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.30.ffn_gate.weight", "offset": 4124538880, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 4159862784, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.30.post_attention_norm.weight", "offset": 4195186688, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 4195201024, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 4195215360, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 4195229696, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.30.attn_output.weight", "offset": 4200275968, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 4210368512, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.30.attn_v.weight", "offset": 4220461056, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.31.attn_k.weight", "offset": 4225507328, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.31.ffn_down.weight", "offset": 4230553600, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 4265877504, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 4301201408, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.31.post_attention_norm.weight", "offset": 4336525312, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 4336539648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 4336553984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 4336568320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 4336582656, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 4346675200, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.31.attn_v.weight", "offset": 4356767744, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.32.ffn_gate.weight", "offset": 4361814016, "shape": [ 3584, 14336 ], "size": 5046272, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 4397137920, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.32.attn_k.weight", "offset": 4432461824, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.32.attn_output.weight", "offset": 4437508096, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 4447600640, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.32.attn_v.weight", "offset": 4457693184, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 4462739456, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4462753792, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.32.post_attention_norm.weight", "offset": 4498077696, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 4498092032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 4498106368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 4498120704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4498135040, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 4533458944, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 4568782848, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.33.post_attention_norm.weight", "offset": 4604106752, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 4604121088, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 4604135424, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 4604149760, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.33.attn_output.weight", "offset": 4609196032, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 4619288576, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.33.attn_v.weight", "offset": 4629381120, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 4634427392, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4634441728, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 4669765632, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.34.ffn_up.weight", "offset": 4705089536, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.34.post_attention_norm.weight", "offset": 4740413440, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 4740427776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 4740442112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 4740456448, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.34.attn_output.weight", "offset": 4745502720, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 4755595264, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.34.attn_v.weight", "offset": 4765687808, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 4770734080, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4770748416, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 4806072320, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 4841396224, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.35.post_attention_norm.weight", "offset": 4876720128, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 4876734464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 4876748800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4876763136, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 4881809408, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 4891901952, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.35.attn_v.weight", "offset": 4901994496, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 4907040768, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4907055104, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 4942379008, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 4977702912, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.36.post_attention_norm.weight", "offset": 5013026816, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 5013041152, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 5013055488, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 5013069824, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.36.attn_output.weight", "offset": 5018116096, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 5028208640, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.36.attn_v.weight", "offset": 5038301184, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 5043347456, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 5043361792, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.37.ffn_gate.weight", "offset": 5078685696, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 5114009600, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.37.post_attention_norm.weight", "offset": 5149333504, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 5149347840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 5149362176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 5149376512, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.37.attn_output.weight", "offset": 5154422784, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 5164515328, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.37.attn_v.weight", "offset": 5174607872, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 5179654144, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5179668480, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 5214992384, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 5250316288, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.38.post_attention_norm.weight", "offset": 5285640192, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 5285654528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 5285668864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 5285683200, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.38.attn_output.weight", "offset": 5290729472, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 5300822016, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.38.attn_v.weight", "offset": 5310914560, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 5315960832, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5315975168, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.39.ffn_gate.weight", "offset": 5351299072, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 5386622976, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.39.post_attention_norm.weight", "offset": 5421946880, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 5421961216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 5421975552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 5421989888, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.39.attn_output.weight", "offset": 5427036160, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 5437128704, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.39.attn_v.weight", "offset": 5447221248, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 5452267520, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 5452281856, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.40.ffn_gate.weight", "offset": 5487605760, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.40.ffn_up.weight", "offset": 5522929664, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.40.post_attention_norm.weight", "offset": 5558253568, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 5558267904, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 5558282240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 5558296576, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.40.attn_output.weight", "offset": 5563342848, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 5573435392, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.40.attn_v.weight", "offset": 5583527936, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 5588574208, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 5588588544, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.41.ffn_gate.weight", "offset": 5623912448, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.41.ffn_up.weight", "offset": 5659236352, "shape": [ 3584, 14336 ], "size": 35323904, "type": 13 }, { "name": "blk.41.post_attention_norm.weight", "offset": 5694560256, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 5694574592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 5694588928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 5694603264, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.41.attn_output.weight", "offset": 5699649536, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 5709742080, "shape": [ 3584, 4096 ], "size": 10092544, "type": 13 }, { "name": "blk.41.attn_v.weight", "offset": 5719834624, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "output_norm.weight", "offset": 5724880896, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "token_embd.weight", "offset": 5724895232, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }