gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 15, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 42162176, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 71063552, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.0.post_attention_norm.weight", "offset": 99964928, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 99979264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 99993600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 100007936, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 104136704, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.0.attn_q.weight", "offset": 112394240, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.0.attn_v.weight", "offset": 120651776, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 126672896, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 126687232, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 168835072, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 197736448, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.1.post_attention_norm.weight", "offset": 226637824, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 226652160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 226666496, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 226680832, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 230809600, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.1.attn_q.weight", "offset": 239067136, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.1.attn_v.weight", "offset": 247324672, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.2.attn_v.weight", "offset": 253345792, "shape": [ 3584, 2048 ], "size": 6021120, "type": 14 }, { "name": "blk.2.ffn_down.weight", "offset": 259366912, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 301514752, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 330416128, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.2.post_attention_norm.weight", "offset": 359317504, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 359331840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 359346176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 359360512, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 363489280, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.2.attn_q.weight", "offset": 371746816, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 380004352, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 380018688, "shape": [ 3584, 2048 ], "size": 14336, "type": 14 }, { "name": "blk.3.ffn_down.weight", "offset": 386039808, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 428187648, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 457089024, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.3.post_attention_norm.weight", "offset": 485990400, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 486004736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 486019072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 486033408, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 490162176, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.3.attn_q.weight", "offset": 498419712, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 506677248, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 506691584, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 506705920, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 548853760, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 577755136, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.4.post_attention_norm.weight", "offset": 606656512, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 606670848, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 606685184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 606699520, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 610828288, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.4.attn_q.weight", "offset": 619085824, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.4.attn_v.weight", "offset": 627343360, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 633364480, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 633378816, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 662280192, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 691181568, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.5.post_attention_norm.weight", "offset": 720082944, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 720097280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 720111616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 720125952, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 724254720, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.5.attn_q.weight", "offset": 732512256, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.5.attn_v.weight", "offset": 740769792, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 744898560, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 744912896, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 773814272, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 802715648, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.6.post_attention_norm.weight", "offset": 831617024, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 831631360, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 831645696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 831660032, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 835788800, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.6.attn_q.weight", "offset": 844046336, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.6.attn_v.weight", "offset": 852303872, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 856432640, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 856446976, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.7.attn_k.weight", "offset": 885348352, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 889477120, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.7.attn_q.weight", "offset": 897734656, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.7.attn_v.weight", "offset": 905992192, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 912013312, "shape": [ 3584, 14336 ], "size": 6021120, "type": 12 }, { "name": "blk.7.ffn_down.weight", "offset": 940914688, "shape": [ 14336, 3584 ], "size": 28901376, "type": 14 }, { "name": "blk.7.post_attention_norm.weight", "offset": 983062528, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 983076864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 983091200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 983105536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 983119872, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 1012021248, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 1040922624, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 1069824000, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 1069838336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1069852672, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 1073981440, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.8.attn_q.weight", "offset": 1082238976, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.8.attn_v.weight", "offset": 1090496512, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1094625280, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1094639616, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_down.weight", "offset": 1098768384, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1127669760, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1127684096, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 1156585472, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1185486848, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1185501184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1185515520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1185529856, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 1189658624, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.9.attn_q.weight", "offset": 1197916160, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.10.attn_q.weight", "offset": 1206173696, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.10.attn_v.weight", "offset": 1214431232, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.10.ffn_down.weight", "offset": 1220452352, "shape": [ 14336, 3584 ], "size": 6021120, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 1262600192, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.10.attn_k.weight", "offset": 1291501568, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1295630336, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1295644672, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1324546048, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1324560384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1324574720, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1332832256, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1332846592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1332860928, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1332875264, "shape": [ 3584, 4096 ], "size": 14336, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1341132800, "shape": [ 3584, 14336 ], "size": 8257536, "type": 12 }, { "name": "blk.11.attn_k.weight", "offset": 1370034176, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1374162944, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1374177280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1374191616, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.attn_v.weight", "offset": 1382449152, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.11.ffn_down.weight", "offset": 1386577920, "shape": [ 14336, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1415479296, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1444380672, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1444395008, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.12.attn_v.weight", "offset": 1473296384, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.12.attn_q.weight", "offset": 1477425152, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 1485682688, "shape": [ 4096, 3584 ], "size": 8257536, "type": 12 }, { "name": "blk.12.attn_k.weight", "offset": 1493940224, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1498068992, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1498083328, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1498097664, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1498112000, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 1527013376, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1555914752, "shape": [ 4096, 3584 ], "size": 28901376, "type": 12 }, { "name": "blk.13.attn_v.weight", "offset": 1564172288, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 1570193408, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1570207744, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1570222080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1570236416, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1599137792, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.13.ffn_down.weight", "offset": 1628039168, "shape": [ 14336, 3584 ], "size": 28901376, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 1670187008, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1670201344, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.13.attn_q.weight", "offset": 1674330112, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1682587648, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1682601984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1682616320, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1711517696, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 1740419072, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.14.post_attention_norm.weight", "offset": 1769320448, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1769334784, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 1777592320, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1777606656, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.14.attn_v.weight", "offset": 1781735424, "shape": [ 3584, 2048 ], "size": 4128768, "type": 12 }, { "name": "blk.14.attn_q.weight", "offset": 1785864192, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.15.attn_v.weight", "offset": 1794121728, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.15.attn_q.weight", "offset": 1798250496, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 1806508032, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1806522368, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1835423744, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 1864325120, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.15.post_attention_norm.weight", "offset": 1893226496, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 1893240832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1893255168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1893269504, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 1897398272, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 1905655808, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 1905670144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1905684480, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 1909813248, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1909827584, "shape": [ 3584, 4096 ], "size": 14336, "type": 12 }, { "name": "blk.16.attn_v.weight", "offset": 1918085120, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 1924106240, "shape": [ 4096, 3584 ], "size": 6021120, "type": 12 }, { "name": "blk.16.post_attention_norm.weight", "offset": 1932363776, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 1932378112, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 1961279488, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.16.ffn_down.weight", "offset": 1990180864, "shape": [ 14336, 3584 ], "size": 28901376, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 2032328704, "shape": [ 3584 ], "size": 42147840, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 2032343040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2032357376, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 2036486144, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 2044743680, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 2044758016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 2044772352, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 2073673728, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.17.ffn_down.weight", "offset": 2102575104, "shape": [ 14336, 3584 ], "size": 28901376, "type": 12 }, { "name": "blk.17.attn_v.weight", "offset": 2131476480, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.17.attn_q.weight", "offset": 2135605248, "shape": [ 3584, 4096 ], "size": 4128768, "type": 12 }, { "name": "blk.18.attn_v.weight", "offset": 2143862784, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2147991552, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2148005888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2148020224, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2176921600, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 2205822976, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.18.post_attention_norm.weight", "offset": 2234724352, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2234738688, "shape": [ 3584, 4096 ], "size": 14336, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 2242996224, "shape": [ 4096, 3584 ], "size": 8257536, "type": 12 }, { "name": "blk.18.attn_k.weight", "offset": 2251253760, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 2255382528, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2255396864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2255411200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2255425536, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 2297573376, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 2326474752, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2355376128, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2355390464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2355404800, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 2359533568, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.19.attn_q.weight", "offset": 2367791104, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.19.attn_v.weight", "offset": 2376048640, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.20.attn_k.weight", "offset": 2382069760, "shape": [ 3584, 2048 ], "size": 6021120, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 2386198528, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.20.attn_q.weight", "offset": 2394456064, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.20.attn_v.weight", "offset": 2402713600, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2406842368, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2406856704, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 2435758080, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 2464659456, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2493560832, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2493575168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2493589504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2493603840, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 2522505216, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2522519552, "shape": [ 3584, 14336 ], "size": 14336, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 2551420928, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.21.post_attention_norm.weight", "offset": 2580322304, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 2580336640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2580350976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2580365312, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 2584494080, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.21.attn_q.weight", "offset": 2592751616, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.21.attn_v.weight", "offset": 2601009152, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 2605137920, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2605152256, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 2647300096, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 2676201472, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.22.post_attention_norm.weight", "offset": 2705102848, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 2705117184, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2705131520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2705145856, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 2709274624, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.22.attn_q.weight", "offset": 2717532160, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.22.attn_v.weight", "offset": 2725789696, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 2731810816, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2731825152, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 2760726528, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 2789627904, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.23.post_attention_norm.weight", "offset": 2818529280, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 2818543616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2818557952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2818572288, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 2822701056, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.23.attn_q.weight", "offset": 2830958592, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.23.attn_v.weight", "offset": 2839216128, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2843344896, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2843359232, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 2872260608, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 2901161984, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.24.post_attention_norm.weight", "offset": 2930063360, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 2930077696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2930092032, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2930106368, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 2934235136, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.24.attn_q.weight", "offset": 2942492672, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.24.attn_v.weight", "offset": 2950750208, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 2954878976, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2954893312, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 2997041152, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 3025942528, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.25.post_attention_norm.weight", "offset": 3054843904, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 3054858240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 3054872576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3054886912, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 3059015680, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.25.attn_q.weight", "offset": 3067273216, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.25.attn_v.weight", "offset": 3075530752, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 3081551872, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3081566208, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 3110467584, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 3139368960, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.26.post_attention_norm.weight", "offset": 3168270336, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 3168284672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 3168299008, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3168313344, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 3172442112, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.26.attn_q.weight", "offset": 3180699648, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.26.attn_v.weight", "offset": 3188957184, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 3193085952, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3193100288, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 3222001664, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 3250903040, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.27.post_attention_norm.weight", "offset": 3279804416, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 3279818752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3279833088, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3279847424, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 3283976192, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.27.attn_q.weight", "offset": 3292233728, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.27.attn_v.weight", "offset": 3300491264, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 3304620032, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3304634368, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.28.ffn_gate.weight", "offset": 3346782208, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 3375683584, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.28.post_attention_norm.weight", "offset": 3404584960, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 3404599296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3404613632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3404627968, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 3408756736, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.28.attn_q.weight", "offset": 3417014272, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.28.attn_v.weight", "offset": 3425271808, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.29.attn_norm.weight", "offset": 3431292928, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3431307264, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.29.ffn_gate.weight", "offset": 3460208640, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 3489110016, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.29.post_attention_norm.weight", "offset": 3518011392, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 3518025728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3518040064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3518054400, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.29.attn_output.weight", "offset": 3522183168, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.29.attn_q.weight", "offset": 3530440704, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.29.attn_v.weight", "offset": 3538698240, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 3542827008, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3542841344, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.30.ffn_gate.weight", "offset": 3571742720, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 3600644096, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.30.post_attention_norm.weight", "offset": 3629545472, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 3629559808, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3629574144, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3629588480, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.30.attn_output.weight", "offset": 3633717248, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.30.attn_q.weight", "offset": 3641974784, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.30.attn_v.weight", "offset": 3650232320, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.31.attn_k.weight", "offset": 3654361088, "shape": [ 3584, 2048 ], "size": 4128768, "type": 12 }, { "name": "blk.31.ffn_down.weight", "offset": 3658489856, "shape": [ 14336, 3584 ], "size": 4128768, "type": 14 }, { "name": "blk.31.ffn_gate.weight", "offset": 3700637696, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 3729539072, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.31.post_attention_norm.weight", "offset": 3758440448, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 3758454784, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3758469120, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 3758483456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3758497792, "shape": [ 4096, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.31.attn_q.weight", "offset": 3766755328, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.31.attn_v.weight", "offset": 3775012864, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.32.ffn_gate.weight", "offset": 3781033984, "shape": [ 3584, 14336 ], "size": 6021120, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 3809935360, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.32.attn_k.weight", "offset": 3838836736, "shape": [ 3584, 2048 ], "size": 28901376, "type": 12 }, { "name": "blk.32.attn_output.weight", "offset": 3842965504, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.32.attn_q.weight", "offset": 3851223040, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.32.attn_v.weight", "offset": 3859480576, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 3863609344, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3863623680, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.32.post_attention_norm.weight", "offset": 3892525056, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 3892539392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 3892553728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 3892568064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 3892582400, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.33.ffn_gate.weight", "offset": 3921483776, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 3950385152, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.33.post_attention_norm.weight", "offset": 3979286528, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 3979300864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 3979315200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 3979329536, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.33.attn_output.weight", "offset": 3983458304, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.33.attn_q.weight", "offset": 3991715840, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.33.attn_v.weight", "offset": 3999973376, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 4004102144, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4004116480, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.34.ffn_gate.weight", "offset": 4046264320, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 4075165696, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.34.post_attention_norm.weight", "offset": 4104067072, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 4104081408, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 4104095744, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 4104110080, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.34.attn_output.weight", "offset": 4108238848, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.34.attn_q.weight", "offset": 4116496384, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.34.attn_v.weight", "offset": 4124753920, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 4130775040, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4130789376, "shape": [ 14336, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.35.ffn_gate.weight", "offset": 4159690752, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 4188592128, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.35.post_attention_norm.weight", "offset": 4217493504, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 4217507840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 4217522176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 4217536512, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 4221665280, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.35.attn_q.weight", "offset": 4229922816, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.35.attn_v.weight", "offset": 4238180352, "shape": [ 3584, 2048 ], "size": 8257536, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 4242309120, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4242323456, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.36.ffn_gate.weight", "offset": 4284471296, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 4313372672, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.36.post_attention_norm.weight", "offset": 4342274048, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 4342288384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 4342302720, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 4342317056, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.36.attn_output.weight", "offset": 4346445824, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.36.attn_q.weight", "offset": 4354703360, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.36.attn_v.weight", "offset": 4362960896, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 4368982016, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4368996352, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.37.ffn_gate.weight", "offset": 4411144192, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 4440045568, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.37.post_attention_norm.weight", "offset": 4468946944, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 4468961280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 4468975616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 4468989952, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.37.attn_output.weight", "offset": 4473118720, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.37.attn_q.weight", "offset": 4481376256, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.37.attn_v.weight", "offset": 4489633792, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 4495654912, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4495669248, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.38.ffn_gate.weight", "offset": 4537817088, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 4566718464, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.38.post_attention_norm.weight", "offset": 4595619840, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 4595634176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 4595648512, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 4595662848, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.38.attn_output.weight", "offset": 4599791616, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.38.attn_q.weight", "offset": 4608049152, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.38.attn_v.weight", "offset": 4616306688, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 4622327808, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 4622342144, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.39.ffn_gate.weight", "offset": 4664489984, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 4693391360, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.39.post_attention_norm.weight", "offset": 4722292736, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 4722307072, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 4722321408, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 4722335744, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.39.attn_output.weight", "offset": 4726464512, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.39.attn_q.weight", "offset": 4734722048, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.39.attn_v.weight", "offset": 4742979584, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.40.attn_norm.weight", "offset": 4749000704, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 4749015040, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.40.ffn_gate.weight", "offset": 4791162880, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.40.ffn_up.weight", "offset": 4820064256, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.40.post_attention_norm.weight", "offset": 4848965632, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 4848979968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 4848994304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 4849008640, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.40.attn_output.weight", "offset": 4853137408, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.40.attn_q.weight", "offset": 4861394944, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.40.attn_v.weight", "offset": 4869652480, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "blk.41.attn_norm.weight", "offset": 4875673600, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 4875687936, "shape": [ 14336, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.41.ffn_gate.weight", "offset": 4917835776, "shape": [ 3584, 14336 ], "size": 42147840, "type": 12 }, { "name": "blk.41.ffn_up.weight", "offset": 4946737152, "shape": [ 3584, 14336 ], "size": 28901376, "type": 12 }, { "name": "blk.41.post_attention_norm.weight", "offset": 4975638528, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 4975652864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 4975667200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 4975681536, "shape": [ 3584, 2048 ], "size": 14336, "type": 12 }, { "name": "blk.41.attn_output.weight", "offset": 4979810304, "shape": [ 4096, 3584 ], "size": 4128768, "type": 12 }, { "name": "blk.41.attn_q.weight", "offset": 4988067840, "shape": [ 3584, 4096 ], "size": 8257536, "type": 12 }, { "name": "blk.41.attn_v.weight", "offset": 4996325376, "shape": [ 3584, 2048 ], "size": 8257536, "type": 14 }, { "name": "output_norm.weight", "offset": 5002346496, "shape": [ 3584 ], "size": 6021120, "type": 0 }, { "name": "token_embd.weight", "offset": 5002360832, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }