gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 14, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 35338240, "shape": [ 3584, 14336 ], "size": 35323904, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 64239616, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.0.post_attention_norm.weight", "offset": 93140992, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 93155328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 93169664, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 93184000, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 97312768, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 105570304, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 113827840, "shape": [ 3584, 2048 ], "size": 8257536, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 118874112, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 118888448, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 154212352, "shape": [ 3584, 14336 ], "size": 35323904, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 183113728, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.1.post_attention_norm.weight", "offset": 212015104, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 212029440, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 212043776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 212058112, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 216186880, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 224444416, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 232701952, "shape": [ 3584, 2048 ], "size": 8257536, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 237748224, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.2.ffn_down.weight", "offset": 242794496, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 278118400, "shape": [ 3584, 14336 ], "size": 35323904, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 307019776, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.2.post_attention_norm.weight", "offset": 335921152, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 335935488, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 335949824, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 335964160, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 340092928, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 348350464, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 356608000, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 356622336, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.3.ffn_down.weight", "offset": 361668608, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 396992512, "shape": [ 3584, 14336 ], "size": 35323904, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 425893888, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.3.post_attention_norm.weight", "offset": 454795264, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 454809600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 454823936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 454838272, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 458967040, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 467224576, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 475482112, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 475496448, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 475510784, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 510834688, "shape": [ 3584, 14336 ], "size": 35323904, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 539736064, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.4.post_attention_norm.weight", "offset": 568637440, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 568651776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 568666112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 568680448, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 572809216, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 581066752, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 589324288, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 593453056, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 593467392, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 622368768, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 651270144, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.17.attn_v.weight\u0002\u0000\u0000\u0000\u0000\u000e", "offset": 680171520, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 680185856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 680200192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 680214528, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 684343296, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 692600832, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 700858368, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 704987136, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 705001472, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 733902848, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 762804224, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.6.post_attention_norm.weight", "offset": 791705600, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 791719936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 791734272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 791748608, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 795877376, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 804134912, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 812392448, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 816521216, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 816535552, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.7.attn_k.weight", "offset": 845436928, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 849565696, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 857823232, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 866080768, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 870209536, "shape": [ 3584, 14336 ], "size": 4128768, "type": 12 }, { "name": "blk.7.ffn_down.weight", "offset": 899110912, "shape": [ 14336, 3584 ], "size": 28901376, "type": 12 }, { "name": "blk.7.post_attention_norm.weight", "offset": 928012288, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 928026624, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 928040960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 928055296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 928069632, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 956971008, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 985872384, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1014773760, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1014788096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1014802432, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 1018931200, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 1027188736, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 1035446272, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1039575040, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1039589376, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_down.weight", "offset": 1043718144, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1072619520, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1072633856, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 1101535232, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1130436608, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1130450944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1130465280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1130479616, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 1134608384, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 1142865920, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 1151123456, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 1159380992, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.10.ffn_down.weight", "offset": 1163509760, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1192411136, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.10.attn_k.weight", "offset": 1221312512, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1225441280, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1225455616, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1254356992, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1254371328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1254385664, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1262643200, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1262657536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1262671872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1262686208, "shape": [ 3584, 4096 ], "size": 14336, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1270943744, "shape": [ 3584, 14336 ], "size": 8257536, "type": 12 }, { "name": "blk.11.attn_k.weight", "offset": 1299845120, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1303973888, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1303988224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1304002560, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 1312260096, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.11.ffn_down.weight", "offset": 1316388864, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1345290240, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1374191616, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1374205952, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 1403107328, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 1407236096, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 1415493632, "shape": [ 4096, 3584 ], "size": 8257536, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1423751168, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1427879936, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1427894272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1427908608, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1427922944, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 1456824320, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1485725696, "shape": [ 4096, 3584 ], "size": 28901376, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 1493983232, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 1498112000, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1498126336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1498140672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1498155008, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1527056384, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.13.ffn_down.weight", "offset": 1555957760, "shape": [ 14336, 3584 ], "size": 28901376, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1584859136, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1584873472, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 1589002240, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1597259776, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1597274112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1597288448, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1626189824, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 1655091200, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.14.post_attention_norm.weight", "offset": 1683992576, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1684006912, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 1692264448, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1692278784, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 1696407552, "shape": [ 3584, 2048 ], "size": 4128768, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 1700536320, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 1708793856, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 1712922624, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 1721180160, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1721194496, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1750095872, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 1778997248, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.15.post_attention_norm.weight", "offset": 1807898624, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 1807912960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1807927296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1807941632, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 1812070400, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 1820327936, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 1820342272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1820356608, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 1824485376, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1824499712, "shape": [ 3584, 4096 ], "size": 14336, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 1832757248, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 1836886016, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.16.post_attention_norm.weight", "offset": 1845143552, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 1845157888, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 1874059264, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.16.ffn_down.weight", "offset": 1902960640, "shape": [ 14336, 3584 ], "size": 28901376, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 1931862016, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 1931876352, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1931890688, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 1936019456, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 1944276992, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 1944291328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 1944305664, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 1973207040, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.17.ffn_down.weight", "offset": 2002108416, "shape": [ 14336, 3584 ], "size": 28901376, "type": 12 }, { "name": "ffn_gate.weight\u0002\u0000\u0000\u0000\u0000", "offset": 2031009792, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 2035138560, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 2043396096, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2047524864, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2047539200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2047553536, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2076454912, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 2105356288, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.18.post_attention_norm.weight", "offset": 2134257664, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2134272000, "shape": [ 3584, 4096 ], "size": 14336, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 2142529536, "shape": [ 4096, 3584 ], "size": 8257536, "type": 12 }, { "name": "blk.18.attn_k.weight", "offset": 2150787072, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 2154915840, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2154930176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2154944512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2154958848, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 2183860224, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 2212761600, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2241662976, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2241677312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2241691648, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 2245820416, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 2254077952, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 2262335488, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.20.attn_k.weight", "offset": 2266464256, "shape": [ 3584, 2048 ], "size": 4128768, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 2270593024, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 2278850560, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 2287108096, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2291236864, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2291251200, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 2320152576, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 2349053952, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2377955328, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2377969664, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2377984000, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2377998336, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 2406899712, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2406914048, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 2435815424, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.21.post_attention_norm.weight", "offset": 2464716800, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 2464731136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2464745472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2464759808, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 2468888576, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 2477146112, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 2485403648, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 2489532416, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2489546752, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2518448128, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 2547349504, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.22.post_attention_norm.weight", "offset": 2576250880, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 2576265216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2576279552, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2576293888, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 2580422656, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 2588680192, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 2596937728, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 2601066496, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2601080832, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 2629982208, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 2658883584, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.23.post_attention_norm.weight", "offset": 2687784960, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 2687799296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2687813632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2687827968, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 2691956736, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 2700214272, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 2708471808, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2712600576, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2712614912, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2741516288, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 2770417664, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.24.post_attention_norm.weight", "offset": 2799319040, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 2799333376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2799347712, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2799362048, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 2803490816, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 2811748352, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 2820005888, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 2824134656, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2824148992, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 2853050368, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 2881951744, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.25.post_attention_norm.weight", "offset": 2910853120, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 2910867456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 2910881792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2910896128, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 2915024896, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 2923282432, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 2931539968, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 2935668736, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2935683072, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 2964584448, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 2993485824, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.26.post_attention_norm.weight", "offset": 3022387200, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 3022401536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3022415872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3022430208, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 3026558976, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 3034816512, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 3043074048, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 3047202816, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3047217152, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 3076118528, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 3105019904, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.27.post_attention_norm.weight", "offset": 3133921280, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 3133935616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3133949952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3133964288, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 3138093056, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 3146350592, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 3154608128, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 3158736896, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3158751232, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.28.ffn_gate.weight", "offset": 3187652608, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 3216553984, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.28.post_attention_norm.weight", "offset": 3245455360, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 3245469696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3245484032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3245498368, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 3249627136, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 3257884672, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 3266142208, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.29.attn_norm.weight", "offset": 3270270976, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3270285312, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 3299186688, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 3328088064, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.29.post_attention_norm.weight", "offset": 3356989440, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 3357003776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3357018112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3357032448, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 3361161216, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 3369418752, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 3377676288, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 3381805056, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3381819392, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "\ufffd?\u0000\ufffd\ufffd?\u0000@\ufffd?\u0000\ufffd\ufffd?\u0000\u0000\ufffd?\u0000\ufffd\ufffd?", "offset": 3410720768, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 3439622144, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.30.post_attention_norm.weight", "offset": 3468523520, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 3468537856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3468552192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3468566528, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 3472695296, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 3480952832, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 3489210368, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 3493339136, "shape": [ 3584, 2048 ], "size": 4128768, "type": 12 }, { "name": "blk.31.ffn_down.weight", "offset": 3497467904, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.31.ffn_gate.weight", "offset": 3526369280, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 3555270656, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.31.post_attention_norm.weight", "offset": 3584172032, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 3584186368, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3584200704, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 3584215040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3584229376, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 3592486912, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 3600744448, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.32.ffn_gate.weight", "offset": 3604873216, "shape": [ 3584, 14336 ], "size": 4128768, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 3633774592, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.32.attn_k.weight", "offset": 3662675968, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 3666804736, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 3675062272, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 3683319808, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 3687448576, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3687462912, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.32.post_attention_norm.weight", "offset": 3716364288, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 3716378624, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 3716392960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 3716407296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 3716421632, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 3745323008, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 3774224384, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.33.post_attention_norm.weight", "offset": 3803125760, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 3803140096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 3803154432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 3803168768, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 3807297536, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 3815555072, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 3823812608, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 3827941376, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 3827955712, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.34.ffn_gate.weight", "offset": 3856857088, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 3885758464, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.34.post_attention_norm.weight", "offset": 3914659840, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 3914674176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 3914688512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 3914702848, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 3918831616, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 3927089152, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 3935346688, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.35.attn_norm.weight", "offset": 3939475456, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 3939489792, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 3968391168, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 3997292544, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.35.post_attention_norm.weight", "offset": 4026193920, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 4026208256, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 4026222592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4026236928, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 4030365696, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 4038623232, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 4046880768, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 4051009536, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4051023872, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.36.ffn_gate.weight", "offset": 4079925248, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 4108826624, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.36.post_attention_norm.weight", "offset": 4137728000, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 4137742336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 4137756672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 4137771008, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 4141899776, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 4150157312, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 4158414848, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 4162543616, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4162557952, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.37.ffn_gate.weight", "offset": 4191459328, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 4220360704, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.37.post_attention_norm.weight", "offset": 4249262080, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 4249276416, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 4249290752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 4249305088, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 4253433856, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 4261691392, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 4269948928, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 4274077696, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4274092032, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.38.ffn_gate.weight", "offset": 4302993408, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 4331894784, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.38.post_attention_norm.weight", "offset": 4360796160, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 4360810496, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 4360824832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 4360839168, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 4364967936, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 4373225472, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 4381483008, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 4385611776, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 4385626112, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.39.ffn_gate.weight", "offset": 4414527488, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 4443428864, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.39.post_attention_norm.weight", "offset": 4472330240, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 4472344576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 4472358912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 4472373248, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 4476502016, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 4484759552, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 4493017088, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.40.attn_norm.weight", "offset": 4497145856, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 4497160192, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.40.ffn_gate.weight", "offset": 4526061568, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 4554962944, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.40.post_attention_norm.weight", "offset": 4583864320, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 4583878656, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 4583892992, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 4583907328, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 4588036096, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 4596293632, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 4604551168, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.41.attn_norm.weight", "offset": 4608679936, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 4608694272, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.41.ffn_gate.weight", "offset": 4637595648, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 4666497024, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.41.post_attention_norm.weight", "offset": 4695398400, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 4695412736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 4695427072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 4695441408, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 4699570176, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 4707827712, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 4716085248, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "output_norm.weight", "offset": 4720214016, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "token_embd.weight", "offset": 4720228352, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }