qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 12, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 234178560, "shape": [ 3584 ], "size": 234178560, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 234192896, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 280870912, "shape": [ 3584, 18944 ], "size": 46678016, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 310044672, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 339218432, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 339232768, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 339234816, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 340023296, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 347248640, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 347262976, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 352782336, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 352784384, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 354045952, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 354060288, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.1.ffn_gate.weight", "offset": 392251392, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 421425152, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 450598912, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 450613248, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 450615296, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 451403776, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 458629120, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 458643456, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 464162816, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 464164864, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 465426432, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 465440768, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 503631872, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 532805632, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 561979392, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 561993728, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 561995776, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 562784256, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 570009600, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 570023936, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 575543296, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 575545344, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 576577536, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 576591872, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 614782976, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 643956736, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 673130496, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 673144832, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 673146880, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 673935360, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 681160704, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 681175040, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 686694400, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 686696448, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 687728640, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 687742976, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 725934080, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 755107840, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 784281600, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 784295936, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 784297984, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 785086464, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 792311808, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 792326144, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 797845504, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000\u0000blk.16.attn_v.", "offset": 797847552, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 798879744, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 798894080, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 837085184, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 866258944, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 895432704, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 895447040, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 895449088, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 896237568, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 903462912, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 903477248, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 908996608, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 908998656, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_k.bias", "offset": 910030848, "shape": [ 512 ], "size": 1032192, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 910032896, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 910821376, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 918046720, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 918061056, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 923580416, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 923582464, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 924614656, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 924628992, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 962820096, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 991993856, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1021167616, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1021181952, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1021184000, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 1021972480, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 1029197824, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1029212160, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 1034731520, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1034733568, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1035765760, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1035780096, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1073971200, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1103144960, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1132318720, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1132333056, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1132335104, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 1133123584, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 1140348928, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1140363264, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 1145882624, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1145884672, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1146916864, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1146931200, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 1185122304, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1214296064, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1243469824, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1243484160, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1243486208, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 1244274688, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 1251500032, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1251514368, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 1257033728, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1257035776, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1258067968, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1258082304, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1296273408, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1325447168, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1354620928, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1354635264, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1354637312, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1355425792, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 1362651136, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1362665472, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 1368184832, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1368186880, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1369219072, "shape": [ 3584, 18944 ], "size": 1032192, "type": 11 }, { "name": "blk.14.attn_k.bias", "offset": 1398392832, "shape": [ 512 ], "size": 29173760, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1398394880, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 1399183360, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 1406408704, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1406423040, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 1411942400, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1411944448, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 1412976640, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1412990976, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 1451182080, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1480355840, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1509529600, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1509543936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1509558272, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 1547749376, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1576923136, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1606096896, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1606111232, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1606113280, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1606901760, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 1614127104, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1614141440, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 1619660800, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1619662848, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1620695040, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1620709376, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 1658900480, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1688074240, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1717248000, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1717262336, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1717264384, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 1718052864, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 1725278208, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1725292544, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 1730811904, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1730813952, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1731846144, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1731860480, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 1770051584, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1799225344, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 1828399104, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1828413440, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1828415488, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 1829203968, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 1836429312, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1836443648, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 1841963008, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1841965056, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1842997248, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1843011584, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 1881202688, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 1910376448, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 1910390784, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1910405120, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 1948596224, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1977769984, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2006943744, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2006958080, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2006960128, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 2007748608, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 2014973952, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2014988288, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 2020507648, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2020509696, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2021541888, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2021556224, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 2059747328, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2088921088, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2118094848, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 2118109184, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2118111232, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 2118899712, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 2126125056, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2126139392, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 2131658752, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "\ufffdoR8Hq\ufffd%@W~\u001d\b\u003c\ufffd̾\u003e\ufffd\u000f", "offset": 2131660800, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2132692992, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2132707328, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 2170898432, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 2200072192, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 2229245952, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 2229260288, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2229262336, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 2230050816, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 2237276160, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2237290496, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 2242809856, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 2242811904, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 2243844096, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2243858432, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2282049536, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2311223296, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 2340397056, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 2340411392, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2340413440, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 2341201920, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 2348427264, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2348441600, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 2353960960, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 2353963008, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 2354995200, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2355009536, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 2393200640, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2422374400, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 2451548160, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 2451562496, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2451564544, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 2452353024, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 2459578368, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2459592704, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 2465112064, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 2465114112, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2466146304, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2466160640, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 2504351744, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2533525504, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 2562699264, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 2562713600, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2562715648, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 2563504128, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 2570729472, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 2570743808, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 2576263168, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 2576265216, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 2577297408, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2577311744, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 2615502848, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2644676608, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 2673850368, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 2673864704, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2673866752, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 2674655232, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 2681880576, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2681894912, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 2687414272, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 2687416320, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2688448512, "shape": [ 3584, 18944 ], "size": 1032192, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2717622272, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.22.attn_k.bias", "offset": 2746796032, "shape": [ 512 ], "size": 29173760, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2746798080, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 2747586560, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 2754811904, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2754826240, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 2760345600, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 2760347648, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "output.weight", "offset": 2761379840, "shape": [ 3584, 152064 ], "size": 1032192, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 3208448000, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3208462336, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 3246653440, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 3246667776, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3246682112, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 3284873216, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 3314046976, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 3343220736, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 3343235072, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3343237120, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 3344025600, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 3351250944, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 3351265280, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 3356784640, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 3356786688, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 3357818880, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3357833216, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 3396024320, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3425198080, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 3454371840, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 3454386176, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3454388224, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 3455176704, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 3462402048, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 3462416384, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 3467935744, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 3467937792, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 3468969984, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3468984320, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 3507175424, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3536349184, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 3565522944, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 3565537280, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3565539328, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 3566327808, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 3573553152, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 3573567488, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 3579086848, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 3579088896, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 3580121088, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3580135424, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 3618326528, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3647500288, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 3676674048, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 3676688384, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3676690432, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 3677478912, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 3684704256, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 3684718592, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 3690237952, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 3690240000, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 3691272192, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3691286528, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 3729477632, "shape": [ 3584, 18944 ], "size": 38191104, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 3758651392, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 3787825152, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 3787839488, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3787841536, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 3788630016, "shape": [ 3584, 3584 ], "size": 788480, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 3795855360, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 3795869696, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 3801389056, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 3801391104, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "output_norm.weight", "offset": 3802423296, "shape": [ 3584 ], "size": 1032192, "type": 0 } ], "version": 3 }