gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 13, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 35338240, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 57415680, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.0.post_attention_norm.weight", "offset": 79493120, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 79507456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 79521792, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 79536128, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 82690048, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.0.attn_q.weight", "offset": 92782592, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.0.attn_v.weight", "offset": 99090432, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 104136704, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 104151040, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 139474944, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 161552384, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.1.post_attention_norm.weight", "offset": 183629824, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 183644160, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 183658496, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 183672832, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 186826752, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.1.attn_q.weight", "offset": 196919296, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.1.attn_v.weight", "offset": 203227136, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.2.attn_v.weight", "offset": 208273408, "shape": [ 3584, 2048 ], "size": 5046272, "type": 13 }, { "name": "blk.2.ffn_down.weight", "offset": 213319680, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 248643584, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 270721024, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.2.post_attention_norm.weight", "offset": 292798464, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 292812800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 292827136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 292841472, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 295995392, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.2.attn_q.weight", "offset": 306087936, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 312395776, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 312410112, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.3.ffn_down.weight", "offset": 317456384, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 352780288, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 374857728, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.3.post_attention_norm.weight", "offset": 396935168, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 396949504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 396963840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 396978176, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 400132096, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.3.attn_q.weight", "offset": 410224640, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 416532480, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 416546816, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 416561152, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 451885056, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 473962496, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.4.post_attention_norm.weight", "offset": 496039936, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 496054272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 496068608, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 496082944, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 499236864, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.4.attn_q.weight", "offset": 509329408, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.4.attn_v.weight", "offset": 515637248, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 520683520, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 520697856, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 556021760, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 578099200, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.17.attn_v.weight\u0002\u0000\u0000\u0000\u0000\u000e", "offset": 600176640, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 600190976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 600205312, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 600219648, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 603373568, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.5.attn_q.weight", "offset": 613466112, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.5.attn_v.weight", "offset": 619773952, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 624820224, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 624834560, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 660158464, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 682235904, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.6.post_attention_norm.weight", "offset": 704313344, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 704327680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 704342016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 704356352, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 707510272, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.6.attn_q.weight", "offset": 717602816, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.6.attn_v.weight", "offset": 723910656, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 728956928, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 728971264, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.7.attn_k.weight", "offset": 751048704, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 754202624, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.7.attn_q.weight", "offset": 764295168, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.7.attn_v.weight", "offset": 770603008, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 775649280, "shape": [ 3584, 14336 ], "size": 5046272, "type": 11 }, { "name": "blk.7.ffn_down.weight", "offset": 797726720, "shape": [ 14336, 3584 ], "size": 22077440, "type": 13 }, { "name": "blk.7.post_attention_norm.weight", "offset": 833050624, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 833064960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 833079296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 833093632, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 833107968, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 868431872, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 890509312, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 912586752, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 912601088, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 912615424, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 915769344, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.8.attn_q.weight", "offset": 925861888, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.8.attn_v.weight", "offset": 932169728, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.8.post_attention_norm.weight", "offset": 937216000, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 937230336, "shape": [ 3584, 2048 ], "size": 14336, "type": 13 }, { "name": "blk.9.ffn_down.weight", "offset": 942276608, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 977600512, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 977614848, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 999692288, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1021769728, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1021784064, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1021798400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1021812736, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 1024966656, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.9.attn_q.weight", "offset": 1035059200, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.10.attn_q.weight", "offset": 1041367040, "shape": [ 3584, 4096 ], "size": 6307840, "type": 11 }, { "name": "blk.10.attn_v.weight", "offset": 1047674880, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.10.ffn_down.weight", "offset": 1052721152, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 1088045056, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.10.attn_k.weight", "offset": 1110122496, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1113276416, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1113290752, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1135368192, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1135382528, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1135396864, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1145489408, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1145503744, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1145518080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1145532416, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1151840256, "shape": [ 3584, 14336 ], "size": 6307840, "type": 11 }, { "name": "blk.11.attn_k.weight", "offset": 1173917696, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 1177071616, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1177085952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1177100288, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.11.attn_v.weight", "offset": 1187192832, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.11.ffn_down.weight", "offset": 1192239104, "shape": [ 14336, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1227563008, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 1249640448, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1249654784, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.12.attn_v.weight", "offset": 1284978688, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.12.attn_q.weight", "offset": 1290024960, "shape": [ 3584, 4096 ], "size": 5046272, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 1296332800, "shape": [ 4096, 3584 ], "size": 6307840, "type": 13 }, { "name": "blk.12.attn_k.weight", "offset": 1306425344, "shape": [ 3584, 2048 ], "size": 10092544, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1309579264, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1309593600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1309607936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1309622272, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 1331699712, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1353777152, "shape": [ 4096, 3584 ], "size": 22077440, "type": 13 }, { "name": "blk.13.attn_v.weight", "offset": 1363869696, "shape": [ 3584, 2048 ], "size": 10092544, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 1368915968, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1368930304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1368944640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1368958976, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 1391036416, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.13.ffn_down.weight", "offset": 1413113856, "shape": [ 14336, 3584 ], "size": 22077440, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 1448437760, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1448452096, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.13.attn_q.weight", "offset": 1451606016, "shape": [ 3584, 4096 ], "size": 3153920, "type": 11 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1457913856, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1457928192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1457942528, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 1493266432, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1515343872, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.14.post_attention_norm.weight", "offset": 1537421312, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1537435648, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 1547528192, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1547542528, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.14.attn_v.weight", "offset": 1550696448, "shape": [ 3584, 2048 ], "size": 3153920, "type": 13 }, { "name": "blk.14.attn_q.weight", "offset": 1555742720, "shape": [ 3584, 4096 ], "size": 5046272, "type": 11 }, { "name": "blk.15.attn_v.weight", "offset": 1562050560, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.15.attn_q.weight", "offset": 1567096832, "shape": [ 3584, 4096 ], "size": 5046272, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 1573404672, "shape": [ 3584 ], "size": 6307840, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1573419008, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 1608742912, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1630820352, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.15.post_attention_norm.weight", "offset": 1652897792, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 1652912128, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1652926464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1652940800, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 1656094720, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 1666187264, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 1666201600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1666215936, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 1669369856, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1669384192, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.16.attn_v.weight", "offset": 1675692032, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 1680738304, "shape": [ 4096, 3584 ], "size": 5046272, "type": 13 }, { "name": "blk.16.post_attention_norm.weight", "offset": 1690830848, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 1690845184, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 1712922624, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.16.ffn_down.weight", "offset": 1735000064, "shape": [ 14336, 3584 ], "size": 22077440, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 1770323968, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 1770338304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1770352640, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 1773506560, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 1783599104, "shape": [ 3584 ], "size": 10092544, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 1783613440, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 1783627776, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 1805705216, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.17.ffn_down.weight", "offset": 1827782656, "shape": [ 14336, 3584 ], "size": 22077440, "type": 13 }, { "name": "ffn_gate.weight\u0002\u0000\u0000\u0000\u0000", "offset": 1863106560, "shape": [ 3584, 2048 ], "size": 35323904, "type": 13 }, { "name": "blk.17.attn_q.weight", "offset": 1868152832, "shape": [ 3584, 4096 ], "size": 5046272, "type": 11 }, { "name": "blk.18.attn_v.weight", "offset": 1874460672, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 1879506944, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 1879521280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1879535616, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 1914859520, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 1936936960, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.18.post_attention_norm.weight", "offset": 1959014400, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 1959028736, "shape": [ 3584, 4096 ], "size": 14336, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 1965336576, "shape": [ 4096, 3584 ], "size": 6307840, "type": 13 }, { "name": "blk.18.attn_k.weight", "offset": 1975429120, "shape": [ 3584, 2048 ], "size": 10092544, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 1978583040, "shape": [ 3584 ], "size": 3153920, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 1978597376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 1978611712, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1978626048, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 2013949952, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2036027392, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2058104832, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2058119168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2058133504, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 2061287424, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.19.attn_q.weight", "offset": 2071379968, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.19.attn_v.weight", "offset": 2077687808, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.20.attn_k.weight", "offset": 2082734080, "shape": [ 3584, 2048 ], "size": 5046272, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 2085888000, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.20.attn_q.weight", "offset": 2095980544, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.20.attn_v.weight", "offset": 2102288384, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 2107334656, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2107348992, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 2142672896, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2164750336, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2186827776, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2186842112, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2186856448, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2186870784, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2222194688, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2222209024, "shape": [ 3584, 14336 ], "size": 14336, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2244286464, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.21.post_attention_norm.weight", "offset": 2266363904, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 2266378240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2266392576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2266406912, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 2269560832, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.21.attn_q.weight", "offset": 2279653376, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.21.attn_v.weight", "offset": 2285961216, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 2291007488, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2291021824, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 2326345728, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2348423168, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.22.post_attention_norm.weight", "offset": 2370500608, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 2370514944, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2370529280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2370543616, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 2373697536, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.22.attn_q.weight", "offset": 2383790080, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.22.attn_v.weight", "offset": 2390097920, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 2395144192, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2395158528, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 2430482432, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2452559872, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.23.post_attention_norm.weight", "offset": 2474637312, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 2474651648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2474665984, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2474680320, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 2477834240, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.23.attn_q.weight", "offset": 2487926784, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.23.attn_v.weight", "offset": 2494234624, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 2499280896, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2499295232, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 2534619136, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 2556696576, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.24.post_attention_norm.weight", "offset": 2578774016, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 2578788352, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2578802688, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2578817024, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 2581970944, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.24.attn_q.weight", "offset": 2592063488, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.24.attn_v.weight", "offset": 2598371328, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 2603417600, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2603431936, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 2638755840, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 2660833280, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.25.post_attention_norm.weight", "offset": 2682910720, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 2682925056, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 2682939392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2682953728, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 2686107648, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.25.attn_q.weight", "offset": 2696200192, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.25.attn_v.weight", "offset": 2702508032, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 2707554304, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2707568640, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 2742892544, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 2764969984, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.26.post_attention_norm.weight", "offset": 2787047424, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 2787061760, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 2787076096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2787090432, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 2790244352, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.26.attn_q.weight", "offset": 2800336896, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.26.attn_v.weight", "offset": 2806644736, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 2811691008, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2811705344, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 2847029248, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 2869106688, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.27.post_attention_norm.weight", "offset": 2891184128, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 2891198464, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 2891212800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 2891227136, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 2894381056, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.27.attn_q.weight", "offset": 2904473600, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.27.attn_v.weight", "offset": 2910781440, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 2915827712, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 2915842048, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.28.ffn_gate.weight", "offset": 2951165952, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 2973243392, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.28.post_attention_norm.weight", "offset": 2995320832, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 2995335168, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 2995349504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 2995363840, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 2998517760, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.28.attn_q.weight", "offset": 3008610304, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.28.attn_v.weight", "offset": 3014918144, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 3019964416, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3019978752, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.29.ffn_gate.weight", "offset": 3055302656, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 3077380096, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.29.post_attention_norm.weight", "offset": 3099457536, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 3099471872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3099486208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3099500544, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.29.attn_output.weight", "offset": 3102654464, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.29.attn_q.weight", "offset": 3112747008, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.29.attn_v.weight", "offset": 3119054848, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 3124101120, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3124115456, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "\ufffd?\u0000\ufffd\ufffd?\u0000@\ufffd?\u0000\ufffd\ufffd?\u0000\u0000\ufffd?\u0000\ufffd\ufffd?", "offset": 3159439360, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 3181516800, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.30.post_attention_norm.weight", "offset": 3203594240, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 3203608576, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3203622912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3203637248, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.30.attn_output.weight", "offset": 3206791168, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.30.attn_q.weight", "offset": 3216883712, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.30.attn_v.weight", "offset": 3223191552, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.31.attn_k.weight", "offset": 3228237824, "shape": [ 3584, 2048 ], "size": 5046272, "type": 11 }, { "name": "blk.31.ffn_down.weight", "offset": 3231391744, "shape": [ 14336, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.31.ffn_gate.weight", "offset": 3266715648, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 3288793088, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.31.post_attention_norm.weight", "offset": 3310870528, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 3310884864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3310899200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 3310913536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3310927872, "shape": [ 4096, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.31.attn_q.weight", "offset": 3321020416, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.31.attn_v.weight", "offset": 3327328256, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.32.ffn_gate.weight", "offset": 3332374528, "shape": [ 3584, 14336 ], "size": 5046272, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 3354451968, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.32.attn_k.weight", "offset": 3376529408, "shape": [ 3584, 2048 ], "size": 22077440, "type": 11 }, { "name": "blk.32.attn_output.weight", "offset": 3379683328, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.32.attn_q.weight", "offset": 3389775872, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.32.attn_v.weight", "offset": 3396083712, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 3401129984, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3401144320, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.32.post_attention_norm.weight", "offset": 3436468224, "shape": [ 3584 ], "size": 35323904, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 3436482560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 3436496896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 3436511232, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 3436525568, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.33.ffn_gate.weight", "offset": 3471849472, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 3493926912, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.33.post_attention_norm.weight", "offset": 3516004352, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 3516018688, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 3516033024, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 3516047360, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.33.attn_output.weight", "offset": 3519201280, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.33.attn_q.weight", "offset": 3529293824, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.33.attn_v.weight", "offset": 3535601664, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 3540647936, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 3540662272, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.34.ffn_gate.weight", "offset": 3575986176, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 3598063616, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.34.post_attention_norm.weight", "offset": 3620141056, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 3620155392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 3620169728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 3620184064, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.34.attn_output.weight", "offset": 3623337984, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.34.attn_q.weight", "offset": 3633430528, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.34.attn_v.weight", "offset": 3639738368, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.35.attn_norm.weight", "offset": 3644784640, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 3644798976, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.35.ffn_gate.weight", "offset": 3680122880, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 3702200320, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.35.post_attention_norm.weight", "offset": 3724277760, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 3724292096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 3724306432, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 3724320768, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 3727474688, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.35.attn_q.weight", "offset": 3737567232, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.35.attn_v.weight", "offset": 3743875072, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 3748921344, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 3748935680, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.36.ffn_gate.weight", "offset": 3784259584, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 3806337024, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.36.post_attention_norm.weight", "offset": 3828414464, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 3828428800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 3828443136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 3828457472, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.36.attn_output.weight", "offset": 3831611392, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.36.attn_q.weight", "offset": 3841703936, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.36.attn_v.weight", "offset": 3848011776, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 3853058048, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 3853072384, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.37.ffn_gate.weight", "offset": 3888396288, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 3910473728, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.37.post_attention_norm.weight", "offset": 3932551168, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 3932565504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 3932579840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 3932594176, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.37.attn_output.weight", "offset": 3935748096, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.37.attn_q.weight", "offset": 3945840640, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.37.attn_v.weight", "offset": 3952148480, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 3957194752, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 3957209088, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.38.ffn_gate.weight", "offset": 3992532992, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 4014610432, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.38.post_attention_norm.weight", "offset": 4036687872, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 4036702208, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 4036716544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 4036730880, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.38.attn_output.weight", "offset": 4039884800, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.38.attn_q.weight", "offset": 4049977344, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.38.attn_v.weight", "offset": 4056285184, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 4061331456, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 4061345792, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.39.ffn_gate.weight", "offset": 4096669696, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 4118747136, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.39.post_attention_norm.weight", "offset": 4140824576, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 4140838912, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 4140853248, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 4140867584, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.39.attn_output.weight", "offset": 4144021504, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.39.attn_q.weight", "offset": 4154114048, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.39.attn_v.weight", "offset": 4160421888, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.40.attn_norm.weight", "offset": 4165468160, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 4165482496, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.40.ffn_gate.weight", "offset": 4200806400, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.40.ffn_up.weight", "offset": 4222883840, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.40.post_attention_norm.weight", "offset": 4244961280, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 4244975616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 4244989952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 4245004288, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.40.attn_output.weight", "offset": 4248158208, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.40.attn_q.weight", "offset": 4258250752, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.40.attn_v.weight", "offset": 4264558592, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "blk.41.attn_norm.weight", "offset": 4269604864, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 4269619200, "shape": [ 14336, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.41.ffn_gate.weight", "offset": 4304943104, "shape": [ 3584, 14336 ], "size": 35323904, "type": 11 }, { "name": "blk.41.ffn_up.weight", "offset": 4327020544, "shape": [ 3584, 14336 ], "size": 22077440, "type": 11 }, { "name": "blk.41.post_attention_norm.weight", "offset": 4349097984, "shape": [ 3584 ], "size": 22077440, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 4349112320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 4349126656, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 4349140992, "shape": [ 3584, 2048 ], "size": 14336, "type": 11 }, { "name": "blk.41.attn_output.weight", "offset": 4352294912, "shape": [ 4096, 3584 ], "size": 3153920, "type": 13 }, { "name": "blk.41.attn_q.weight", "offset": 4362387456, "shape": [ 3584, 4096 ], "size": 10092544, "type": 11 }, { "name": "blk.41.attn_v.weight", "offset": 4368695296, "shape": [ 3584, 2048 ], "size": 6307840, "type": 13 }, { "name": "output_norm.weight", "offset": 4373741568, "shape": [ 3584 ], "size": 5046272, "type": 0 }, { "name": "token_embd.weight", "offset": 4373755904, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }