gemma2

{ "metadata": { "gemma2.attention.head_count": 16, "gemma2.attention.head_count_kv": 8, "gemma2.attention.key_length": 256, "gemma2.attention.layer_norm_rms_epsilon": 0.000001, "gemma2.attention.value_length": 256, "gemma2.block_count": 42, "gemma2.context_length": 8192, "gemma2.embedding_length": 3584, "gemma2.feed_forward_length": 14336, "general.architecture": "gemma2", "general.file_type": 2, "general.name": "gemma2", "general.quantization_version": 2, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.add_padding_token": false, "tokenizer.ggml.add_unknown_token": false, "tokenizer.ggml.bos_token_id": 2, "tokenizer.ggml.eos_token_id": 1, "tokenizer.ggml.eot_token_id": 107, "tokenizer.ggml.middle_token_id": 68, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 0, "tokenizer.ggml.pre": "default", "tokenizer.ggml.prefix_token_id": 67, "tokenizer.ggml.scores": "... (256000 values)", "tokenizer.ggml.suffix_token_id": 69, "tokenizer.ggml.token_type": "... (256000 values)", "tokenizer.ggml.tokens": "... (256000 values)", "tokenizer.ggml.unknown_token_id": 3 }, "num_params": 9241705984, "tensors": [ { "name": "blk.0.ffn_down.weight", "offset": 14336, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 28915712, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 57817088, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.0.post_attention_norm.weight", "offset": 86718464, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.0.post_ffw_norm.weight", "offset": 86732800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.ffn_norm.weight", "offset": 86747136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 86761472, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 90890240, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.0.attn_q.weight", "offset": 99147776, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.0.attn_v.weight", "offset": 107405312, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 111534080, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 111548416, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 140449792, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 169351168, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.1.post_attention_norm.weight", "offset": 198252544, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.1.post_ffw_norm.weight", "offset": 198266880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.ffn_norm.weight", "offset": 198281216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 198295552, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 202424320, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.1.attn_q.weight", "offset": 210681856, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.1.attn_v.weight", "offset": 218939392, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.2.attn_v.weight", "offset": 223068160, "shape": [ 3584, 2048 ], "size": 4128768, "type": 2 }, { "name": "blk.2.ffn_down.weight", "offset": 227196928, "shape": [ 14336, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 256098304, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 284999680, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.2.post_attention_norm.weight", "offset": 313901056, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.2.post_ffw_norm.weight", "offset": 313915392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.ffn_norm.weight", "offset": 313929728, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 313944064, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 318072832, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.2.attn_q.weight", "offset": 326330368, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 334587904, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 334602240, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.3.ffn_down.weight", "offset": 338731008, "shape": [ 14336, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 367632384, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 396533760, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.3.post_attention_norm.weight", "offset": 425435136, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.3.post_ffw_norm.weight", "offset": 425449472, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.ffn_norm.weight", "offset": 425463808, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 425478144, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 429606912, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.3.attn_q.weight", "offset": 437864448, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 446121984, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.4.ffn_norm.weight", "offset": 446136320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 446150656, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 475052032, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 503953408, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.4.post_attention_norm.weight", "offset": 532854784, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.4.post_ffw_norm.weight", "offset": 532869120, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_norm.weight", "offset": 532883456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 532897792, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 537026560, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.4.attn_q.weight", "offset": 545284096, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.4.attn_v.weight", "offset": 553541632, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 557670400, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 557684736, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 586586112, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 615487488, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.5.post_attention_norm.weight", "offset": 644388864, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.5.post_ffw_norm.weight", "offset": 644403200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.ffn_norm.weight", "offset": 644417536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 644431872, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 648560640, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.5.attn_q.weight", "offset": 656818176, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.5.attn_v.weight", "offset": 665075712, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 669204480, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 669218816, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 698120192, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 727021568, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.6.post_attention_norm.weight", "offset": 755922944, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.6.post_ffw_norm.weight", "offset": 755937280, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.ffn_norm.weight", "offset": 755951616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 755965952, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 760094720, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.6.attn_q.weight", "offset": 768352256, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.6.attn_v.weight", "offset": 776609792, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 780738560, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.7.ffn_up.weight", "offset": 780752896, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.7.attn_k.weight", "offset": 809654272, "shape": [ 3584, 2048 ], "size": 28901376, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 813783040, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.7.attn_q.weight", "offset": 822040576, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.7.attn_v.weight", "offset": 830298112, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 834426880, "shape": [ 3584, 14336 ], "size": 4128768, "type": 2 }, { "name": "blk.7.ffn_down.weight", "offset": 863328256, "shape": [ 14336, 3584 ], "size": 28901376, "type": 2 }, { "name": "blk.7.post_attention_norm.weight", "offset": 892229632, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.7.post_ffw_norm.weight", "offset": 892243968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_norm.weight", "offset": 892258304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_norm.weight", "offset": 892272640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 892286976, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 921188352, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 950089728, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.8.post_ffw_norm.weight", "offset": 978991104, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.8.ffn_norm.weight", "offset": 979005440, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 979019776, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 983148544, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.8.attn_q.weight", "offset": 991406080, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.8.attn_v.weight", "offset": 999663616, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.8.post_attention_norm.weight", "offset": 1003792384, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1003806720, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.9.ffn_down.weight", "offset": 1007935488, "shape": [ 14336, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 1036836864, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.9.ffn_gate.weight", "offset": 1036851200, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 1065752576, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.9.post_attention_norm.weight", "offset": 1094653952, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.9.post_ffw_norm.weight", "offset": 1094668288, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.ffn_norm.weight", "offset": 1094682624, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1094696960, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 1098825728, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.9.attn_q.weight", "offset": 1107083264, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.10.attn_q.weight", "offset": 1115340800, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.10.attn_v.weight", "offset": 1123598336, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.10.ffn_down.weight", "offset": 1127727104, "shape": [ 14336, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 1156628480, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.10.attn_k.weight", "offset": 1185529856, "shape": [ 3584, 2048 ], "size": 28901376, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1189658624, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.10.ffn_up.weight", "offset": 1189672960, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.10.post_attention_norm.weight", "offset": 1218574336, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.10.post_ffw_norm.weight", "offset": 1218588672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1218603008, "shape": [ 4096, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1226860544, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.11.ffn_norm.weight", "offset": 1226874880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.post_attention_norm.weight", "offset": 1226889216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1226903552, "shape": [ 3584, 4096 ], "size": 14336, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1235161088, "shape": [ 3584, 14336 ], "size": 8257536, "type": 2 }, { "name": "blk.11.attn_k.weight", "offset": 1264062464, "shape": [ 3584, 2048 ], "size": 28901376, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 1268191232, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.11.post_ffw_norm.weight", "offset": 1268205568, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1268219904, "shape": [ 4096, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.11.attn_v.weight", "offset": 1276477440, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.11.ffn_down.weight", "offset": 1280606208, "shape": [ 14336, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 1309507584, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1338408960, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1338423296, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.12.attn_v.weight", "offset": 1367324672, "shape": [ 3584, 2048 ], "size": 28901376, "type": 2 }, { "name": "blk.12.attn_q.weight", "offset": 1371453440, "shape": [ 3584, 4096 ], "size": 4128768, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 1379710976, "shape": [ 4096, 3584 ], "size": 8257536, "type": 2 }, { "name": "blk.12.attn_k.weight", "offset": 1387968512, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 1392097280, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.12.post_ffw_norm.weight", "offset": 1392111616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.post_attention_norm.weight", "offset": 1392125952, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.12.ffn_up.weight", "offset": 1392140288, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 1421041664, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 1449943040, "shape": [ 4096, 3584 ], "size": 28901376, "type": 2 }, { "name": "blk.13.attn_v.weight", "offset": 1458200576, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 1462329344, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.13.post_ffw_norm.weight", "offset": 1462343680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.post_attention_norm.weight", "offset": 1462358016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.13.ffn_up.weight", "offset": 1462372352, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 1491273728, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.13.ffn_down.weight", "offset": 1520175104, "shape": [ 14336, 3584 ], "size": 28901376, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 1549076480, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1549090816, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.13.attn_q.weight", "offset": 1553219584, "shape": [ 3584, 4096 ], "size": 4128768, "type": 2 }, { "name": "blk.14.post_ffw_norm.weight", "offset": 1561477120, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.14.attn_norm.weight", "offset": 1561491456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1561505792, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 1590407168, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 1619308544, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.14.post_attention_norm.weight", "offset": 1648209920, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1648224256, "shape": [ 4096, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 1656481792, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1656496128, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.14.attn_v.weight", "offset": 1660624896, "shape": [ 3584, 2048 ], "size": 4128768, "type": 2 }, { "name": "blk.14.attn_q.weight", "offset": 1664753664, "shape": [ 3584, 4096 ], "size": 4128768, "type": 2 }, { "name": "blk.15.attn_v.weight", "offset": 1673011200, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.15.attn_q.weight", "offset": 1677139968, "shape": [ 3584, 4096 ], "size": 4128768, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 1685397504, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1685411840, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 1714313216, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 1743214592, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.15.post_attention_norm.weight", "offset": 1772115968, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.15.post_ffw_norm.weight", "offset": 1772130304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_norm.weight", "offset": 1772144640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1772158976, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 1776287744, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 1784545280, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.attn_norm.weight", "offset": 1784559616, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1784573952, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.16.post_ffw_norm.weight", "offset": 1788702720, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1788717056, "shape": [ 3584, 4096 ], "size": 14336, "type": 2 }, { "name": "blk.16.attn_v.weight", "offset": 1796974592, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 1801103360, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.16.post_attention_norm.weight", "offset": 1809360896, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.16.ffn_up.weight", "offset": 1809375232, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 1838276608, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.16.ffn_down.weight", "offset": 1867177984, "shape": [ 14336, 3584 ], "size": 28901376, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 1896079360, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.17.ffn_norm.weight", "offset": 1896093696, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1896108032, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 1900236800, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.17.post_ffw_norm.weight", "offset": 1908494336, "shape": [ 3584 ], "size": 8257536, "type": 0 }, { "name": "blk.17.post_attention_norm.weight", "offset": 1908508672, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.17.ffn_up.weight", "offset": 1908523008, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 1937424384, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.17.ffn_down.weight", "offset": 1966325760, "shape": [ 14336, 3584 ], "size": 28901376, "type": 2 }, { "name": "blk.17.attn_v.weight", "offset": 1995227136, "shape": [ 3584, 2048 ], "size": 28901376, "type": 2 }, { "name": "blk.17.attn_q.weight", "offset": 1999355904, "shape": [ 3584, 4096 ], "size": 4128768, "type": 2 }, { "name": "blk.18.attn_v.weight", "offset": 2007613440, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.18.post_ffw_norm.weight", "offset": 2011742208, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.18.attn_norm.weight", "offset": 2011756544, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2011770880, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 2040672256, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 2069573632, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.18.post_attention_norm.weight", "offset": 2098475008, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2098489344, "shape": [ 3584, 4096 ], "size": 14336, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 2106746880, "shape": [ 4096, 3584 ], "size": 8257536, "type": 2 }, { "name": "blk.18.attn_k.weight", "offset": 2115004416, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 2119133184, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.19.ffn_norm.weight", "offset": 2119147520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_norm.weight", "offset": 2119161856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2119176192, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 2148077568, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 2176978944, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.19.post_attention_norm.weight", "offset": 2205880320, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.19.post_ffw_norm.weight", "offset": 2205894656, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2205908992, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 2210037760, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.19.attn_q.weight", "offset": 2218295296, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.19.attn_v.weight", "offset": 2226552832, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.20.attn_k.weight", "offset": 2230681600, "shape": [ 3584, 2048 ], "size": 4128768, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 2234810368, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.20.attn_q.weight", "offset": 2243067904, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.20.attn_v.weight", "offset": 2251325440, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 2255454208, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2255468544, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 2284369920, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 2313271296, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.20.post_attention_norm.weight", "offset": 2342172672, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.20.post_ffw_norm.weight", "offset": 2342187008, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.20.ffn_norm.weight", "offset": 2342201344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2342215680, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 2371117056, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.21.ffn_gate.weight", "offset": 2371131392, "shape": [ 3584, 14336 ], "size": 14336, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 2400032768, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.21.post_attention_norm.weight", "offset": 2428934144, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.21.post_ffw_norm.weight", "offset": 2428948480, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.ffn_norm.weight", "offset": 2428962816, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2428977152, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 2433105920, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.21.attn_q.weight", "offset": 2441363456, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.21.attn_v.weight", "offset": 2449620992, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 2453749760, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2453764096, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 2482665472, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 2511566848, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.22.post_attention_norm.weight", "offset": 2540468224, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.22.post_ffw_norm.weight", "offset": 2540482560, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.ffn_norm.weight", "offset": 2540496896, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2540511232, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 2544640000, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.22.attn_q.weight", "offset": 2552897536, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.22.attn_v.weight", "offset": 2561155072, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 2565283840, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2565298176, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 2594199552, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 2623100928, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.23.post_attention_norm.weight", "offset": 2652002304, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.23.post_ffw_norm.weight", "offset": 2652016640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_norm.weight", "offset": 2652030976, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2652045312, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 2656174080, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.23.attn_q.weight", "offset": 2664431616, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.23.attn_v.weight", "offset": 2672689152, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 2676817920, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2676832256, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 2705733632, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 2734635008, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.24.post_attention_norm.weight", "offset": 2763536384, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.24.post_ffw_norm.weight", "offset": 2763550720, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.ffn_norm.weight", "offset": 2763565056, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2763579392, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 2767708160, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.24.attn_q.weight", "offset": 2775965696, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.24.attn_v.weight", "offset": 2784223232, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 2788352000, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2788366336, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 2817267712, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 2846169088, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.25.post_attention_norm.weight", "offset": 2875070464, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.25.post_ffw_norm.weight", "offset": 2875084800, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.ffn_norm.weight", "offset": 2875099136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2875113472, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 2879242240, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.25.attn_q.weight", "offset": 2887499776, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.25.attn_v.weight", "offset": 2895757312, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 2899886080, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2899900416, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 2928801792, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 2957703168, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.26.post_attention_norm.weight", "offset": 2986604544, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.26.post_ffw_norm.weight", "offset": 2986618880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.ffn_norm.weight", "offset": 2986633216, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2986647552, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 2990776320, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.26.attn_q.weight", "offset": 2999033856, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.26.attn_v.weight", "offset": 3007291392, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 3011420160, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3011434496, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 3040335872, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 3069237248, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.27.post_attention_norm.weight", "offset": 3098138624, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.27.post_ffw_norm.weight", "offset": 3098152960, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.ffn_norm.weight", "offset": 3098167296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3098181632, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 3102310400, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.27.attn_q.weight", "offset": 3110567936, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.27.attn_v.weight", "offset": 3118825472, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 3122954240, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3122968576, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.28.ffn_gate.weight", "offset": 3151869952, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 3180771328, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.28.post_attention_norm.weight", "offset": 3209672704, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.28.post_ffw_norm.weight", "offset": 3209687040, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.ffn_norm.weight", "offset": 3209701376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.28.attn_k.weight", "offset": 3209715712, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 3213844480, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.28.attn_q.weight", "offset": 3222102016, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.28.attn_v.weight", "offset": 3230359552, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 3234488320, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3234502656, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.29.ffn_gate.weight", "offset": 3263404032, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 3292305408, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.29.post_attention_norm.weight", "offset": 3321206784, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.29.post_ffw_norm.weight", "offset": 3321221120, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.ffn_norm.weight", "offset": 3321235456, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.29.attn_k.weight", "offset": 3321249792, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.29.attn_output.weight", "offset": 3325378560, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.29.attn_q.weight", "offset": 3333636096, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.29.attn_v.weight", "offset": 3341893632, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 3346022400, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3346036736, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.30.ffn_gate.weight", "offset": 3374938112, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 3403839488, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.30.post_attention_norm.weight", "offset": 3432740864, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.30.post_ffw_norm.weight", "offset": 3432755200, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.ffn_norm.weight", "offset": 3432769536, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.30.attn_k.weight", "offset": 3432783872, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.30.attn_output.weight", "offset": 3436912640, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.30.attn_q.weight", "offset": 3445170176, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.30.attn_v.weight", "offset": 3453427712, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.31.attn_k.weight", "offset": 3457556480, "shape": [ 3584, 2048 ], "size": 4128768, "type": 2 }, { "name": "blk.31.ffn_down.weight", "offset": 3461685248, "shape": [ 14336, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.31.ffn_gate.weight", "offset": 3490586624, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 3519488000, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.31.post_attention_norm.weight", "offset": 3548389376, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.31.post_ffw_norm.weight", "offset": 3548403712, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.ffn_norm.weight", "offset": 3548418048, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_norm.weight", "offset": 3548432384, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3548446720, "shape": [ 4096, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.31.attn_q.weight", "offset": 3556704256, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.31.attn_v.weight", "offset": 3564961792, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.32.ffn_gate.weight", "offset": 3569090560, "shape": [ 3584, 14336 ], "size": 4128768, "type": 2 }, { "name": "blk.32.ffn_up.weight", "offset": 3597991936, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.32.attn_k.weight", "offset": 3626893312, "shape": [ 3584, 2048 ], "size": 28901376, "type": 2 }, { "name": "blk.32.attn_output.weight", "offset": 3631022080, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.32.attn_q.weight", "offset": 3639279616, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.32.attn_v.weight", "offset": 3647537152, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 3651665920, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 3651680256, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.32.post_attention_norm.weight", "offset": 3680581632, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.32.post_ffw_norm.weight", "offset": 3680595968, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.32.ffn_norm.weight", "offset": 3680610304, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_norm.weight", "offset": 3680624640, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 3680638976, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.33.ffn_gate.weight", "offset": 3709540352, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.33.ffn_up.weight", "offset": 3738441728, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.33.post_attention_norm.weight", "offset": 3767343104, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.33.post_ffw_norm.weight", "offset": 3767357440, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.ffn_norm.weight", "offset": 3767371776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.33.attn_k.weight", "offset": 3767386112, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.33.attn_output.weight", "offset": 3771514880, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.33.attn_q.weight", "offset": 3779772416, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.33.attn_v.weight", "offset": 3788029952, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 3792158720, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 3792173056, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.34.ffn_gate.weight", "offset": 3821074432, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.34.ffn_up.weight", "offset": 3849975808, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.34.post_attention_norm.weight", "offset": 3878877184, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.34.post_ffw_norm.weight", "offset": 3878891520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.ffn_norm.weight", "offset": 3878905856, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.34.attn_k.weight", "offset": 3878920192, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.34.attn_output.weight", "offset": 3883048960, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.34.attn_q.weight", "offset": 3891306496, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.34.attn_v.weight", "offset": 3899564032, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.35.attn_norm.weight", "offset": 3903692800, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 3903707136, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.35.ffn_gate.weight", "offset": 3932608512, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.35.ffn_up.weight", "offset": 3961509888, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.35.post_attention_norm.weight", "offset": 3990411264, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.35.post_ffw_norm.weight", "offset": 3990425600, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.ffn_norm.weight", "offset": 3990439936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.35.attn_k.weight", "offset": 3990454272, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.35.attn_output.weight", "offset": 3994583040, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.35.attn_q.weight", "offset": 4002840576, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.35.attn_v.weight", "offset": 4011098112, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.36.ffn_norm.weight", "offset": 4015226880, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4015241216, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.36.ffn_gate.weight", "offset": 4044142592, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.36.ffn_up.weight", "offset": 4073043968, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.36.post_attention_norm.weight", "offset": 4101945344, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.36.post_ffw_norm.weight", "offset": 4101959680, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 4101974016, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.36.attn_k.weight", "offset": 4101988352, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.36.attn_output.weight", "offset": 4106117120, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.36.attn_q.weight", "offset": 4114374656, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.36.attn_v.weight", "offset": 4122632192, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.37.attn_norm.weight", "offset": 4126760960, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4126775296, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.37.ffn_gate.weight", "offset": 4155676672, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.37.ffn_up.weight", "offset": 4184578048, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.37.post_attention_norm.weight", "offset": 4213479424, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.37.post_ffw_norm.weight", "offset": 4213493760, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.ffn_norm.weight", "offset": 4213508096, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.37.attn_k.weight", "offset": 4213522432, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.37.attn_output.weight", "offset": 4217651200, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.37.attn_q.weight", "offset": 4225908736, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.37.attn_v.weight", "offset": 4234166272, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.38.attn_norm.weight", "offset": 4238295040, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4238309376, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.38.ffn_gate.weight", "offset": 4267210752, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.38.ffn_up.weight", "offset": 4296112128, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.38.post_attention_norm.weight", "offset": 4325013504, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.38.post_ffw_norm.weight", "offset": 4325027840, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.ffn_norm.weight", "offset": 4325042176, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.38.attn_k.weight", "offset": 4325056512, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.38.attn_output.weight", "offset": 4329185280, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.38.attn_q.weight", "offset": 4337442816, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.38.attn_v.weight", "offset": 4345700352, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 4349829120, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 4349843456, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.39.ffn_gate.weight", "offset": 4378744832, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.39.ffn_up.weight", "offset": 4407646208, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.39.post_attention_norm.weight", "offset": 4436547584, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.39.post_ffw_norm.weight", "offset": 4436561920, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.ffn_norm.weight", "offset": 4436576256, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.39.attn_k.weight", "offset": 4436590592, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.39.attn_output.weight", "offset": 4440719360, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.39.attn_q.weight", "offset": 4448976896, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.39.attn_v.weight", "offset": 4457234432, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.40.attn_norm.weight", "offset": 4461363200, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.40.ffn_down.weight", "offset": 4461377536, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.40.ffn_gate.weight", "offset": 4490278912, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.40.ffn_up.weight", "offset": 4519180288, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.40.post_attention_norm.weight", "offset": 4548081664, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.40.post_ffw_norm.weight", "offset": 4548096000, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.ffn_norm.weight", "offset": 4548110336, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.40.attn_k.weight", "offset": 4548124672, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.40.attn_output.weight", "offset": 4552253440, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.40.attn_q.weight", "offset": 4560510976, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.40.attn_v.weight", "offset": 4568768512, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "blk.41.attn_norm.weight", "offset": 4572897280, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "blk.41.ffn_down.weight", "offset": 4572911616, "shape": [ 14336, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.41.ffn_gate.weight", "offset": 4601812992, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.41.ffn_up.weight", "offset": 4630714368, "shape": [ 3584, 14336 ], "size": 28901376, "type": 2 }, { "name": "blk.41.post_attention_norm.weight", "offset": 4659615744, "shape": [ 3584 ], "size": 28901376, "type": 0 }, { "name": "blk.41.post_ffw_norm.weight", "offset": 4659630080, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.ffn_norm.weight", "offset": 4659644416, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.41.attn_k.weight", "offset": 4659658752, "shape": [ 3584, 2048 ], "size": 14336, "type": 2 }, { "name": "blk.41.attn_output.weight", "offset": 4663787520, "shape": [ 4096, 3584 ], "size": 4128768, "type": 2 }, { "name": "blk.41.attn_q.weight", "offset": 4672045056, "shape": [ 3584, 4096 ], "size": 8257536, "type": 2 }, { "name": "blk.41.attn_v.weight", "offset": 4680302592, "shape": [ 3584, 2048 ], "size": 8257536, "type": 2 }, { "name": "output_norm.weight", "offset": 4684431360, "shape": [ 3584 ], "size": 4128768, "type": 0 }, { "name": "token_embd.weight", "offset": 4684445696, "shape": [ 3584, 256000 ], "size": 14336, "type": 14 } ], "version": 3 }