qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 13, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 234178560, "shape": [ 3584 ], "size": 234178560, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 234192896, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 280870912, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 310044672, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 339218432, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 339232768, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 339234816, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 340023296, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 348854272, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 348868608, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 354387968, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 354390016, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 355651584, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 355665920, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 402343936, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 431517696, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 460691456, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 460705792, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 460707840, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 461496320, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 470327296, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 470341632, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 475860992, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 475863040, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 477124608, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 477138944, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 523816960, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 552990720, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 582164480, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 582178816, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 582180864, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 582969344, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 591800320, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 591814656, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 597334016, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 597336064, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 598597632, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 598611968, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 645289984, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 674463744, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 703637504, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 703651840, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 703653888, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 704442368, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 713273344, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 713287680, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 718807040, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 718809088, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 720070656, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 720084992, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 766763008, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 795936768, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 825110528, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 825124864, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 825126912, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 825915392, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 834746368, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 834760704, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 840280064, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 840282112, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 841543680, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 841558016, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 888236032, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 917409792, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 946583552, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 946597888, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 946599936, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 947388416, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 956219392, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 956233728, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 961753088, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 961755136, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.6.attn_k.bias", "offset": 963016704, "shape": [ 512 ], "size": 1261568, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 963018752, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 963807232, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 972638208, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 972652544, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 978171904, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 978173952, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 979435520, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 979449856, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 1026127872, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1055301632, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1084475392, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1084489728, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1084491776, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 1085280256, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 1094111232, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1094125568, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 1099644928, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1099646976, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1100908544, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1100922880, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1147600896, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1176774656, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1205948416, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1205962752, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1205964800, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 1206753280, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 1215584256, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1215598592, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 1221117952, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1221120000, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1222381568, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1222395904, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 1269073920, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1298247680, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1327421440, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1327435776, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1327437824, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 1328226304, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 1337057280, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1337071616, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 1342590976, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1342593024, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 1343854592, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1343868928, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1390546944, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1419720704, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1448894464, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1448908800, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1448910848, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1449699328, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 1458530304, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1458544640, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 1464064000, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1464066048, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 1465327616, "shape": [ 3584, 18944 ], "size": 1261568, "type": 11 }, { "name": "blk.14.attn_k.bias", "offset": 1494501376, "shape": [ 512 ], "size": 29173760, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1494503424, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 1495291904, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 1504122880, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1504137216, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 1509656576, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1509658624, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 1510920192, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1510934528, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 1557612544, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1586786304, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1615960064, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1615974400, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1615988736, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 1662666752, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1691840512, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1721014272, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1721028608, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1721030656, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1721819136, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 1730650112, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1730664448, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 1736183808, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1736185856, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 1737447424, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1737461760, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 1784139776, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1813313536, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1842487296, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1842501632, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1842503680, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 1843292160, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 1852123136, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1852137472, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 1857656832, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1857658880, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1858920448, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1858934784, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 1905612800, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1934786560, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 1963960320, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1963974656, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1963976704, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 1964765184, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 1973596160, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1973610496, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 1979129856, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1979131904, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 1980393472, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1980407808, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 2027085824, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2056259584, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 2056273920, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2056288256, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 2102966272, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 2132140032, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2161313792, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2161328128, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2161330176, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 2162118656, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 2170949632, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2170963968, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 2176483328, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2176485376, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 2177746944, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2177761280, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 2224439296, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2253613056, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2282786816, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 2282801152, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2282803200, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 2283591680, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 2292422656, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2292436992, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 2297956352, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 2297958400, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 2299219968, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2299234304, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 2345912320, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 2375086080, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 2404259840, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 2404274176, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2404276224, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 2405064704, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 2413895680, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2413910016, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 2419429376, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 2419431424, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 2420692992, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2420707328, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 2467385344, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2496559104, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 2525732864, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 2525747200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2525749248, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 2526537728, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 2535368704, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2535383040, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 2540902400, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 2540904448, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 2542166016, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2542180352, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 2588858368, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2618032128, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 2647205888, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 2647220224, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2647222272, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 2648010752, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 2656841728, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2656856064, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 2662375424, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 2662377472, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 2663639040, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2663653376, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 2710331392, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2739505152, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 2768678912, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 2768693248, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2768695296, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 2769483776, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 2778314752, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 2778329088, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 2783848448, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 2783850496, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 2785112064, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2785126400, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 2831804416, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2860978176, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 2890151936, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 2890166272, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2890168320, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 2890956800, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 2899787776, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2899802112, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 2905321472, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 2905323520, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 2906585088, "shape": [ 3584, 18944 ], "size": 1261568, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2935758848, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.22.attn_k.bias", "offset": 2964932608, "shape": [ 512 ], "size": 29173760, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2964934656, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 2965723136, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 2974554112, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2974568448, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 2980087808, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 2980089856, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "output.weight", "offset": 2981351424, "shape": [ 3584, 152064 ], "size": 1261568, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 3428419584, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3428433920, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 3475111936, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 3475126272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3475140608, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 3521818624, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 3550992384, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 3580166144, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 3580180480, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3580182528, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 3580971008, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 3589801984, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 3589816320, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 3595335680, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 3595337728, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 3596599296, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3596613632, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 3643291648, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3672465408, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 3701639168, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 3701653504, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3701655552, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 3702444032, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 3711275008, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 3711289344, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 3716808704, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 3716810752, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 3718072320, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3718086656, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 3764764672, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3793938432, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 3823112192, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 3823126528, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3823128576, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 3823917056, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 3832748032, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 3832762368, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 3838281728, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 3838283776, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 3839545344, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3839559680, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 3886237696, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3915411456, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 3944585216, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 3944599552, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3944601600, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 3945390080, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 3954221056, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 3954235392, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 3959754752, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 3959756800, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 3961018368, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3961032704, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 4007710720, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 4036884480, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 4066058240, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 4066072576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4066074624, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 4066863104, "shape": [ 3584, 3584 ], "size": 788480, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 4075694080, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 4075708416, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 4081227776, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 4081229824, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "output_norm.weight", "offset": 4082491392, "shape": [ 3584 ], "size": 1261568, "type": 0 } ], "version": 3 }