qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 14, "general.name": "Qwen2-7B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 306561024, "shape": [ 3584 ], "size": 306561024, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 306575360, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 353253376, "shape": [ 3584, 18944 ], "size": 46678016, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 391444480, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 429635584, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 429649920, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 429651968, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 430684160, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 437909504, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 437923840, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.0.attn_v.bias", "offset": 445149184, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 445151232, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 446412800, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 446427136, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 493105152, "shape": [ 3584, 18944 ], "size": 46678016, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 531296256, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 569487360, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 569501696, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 569503744, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 570535936, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 577761280, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 577775616, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.1.attn_v.bias", "offset": 585000960, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 585003008, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 586264576, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 586278912, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 632956928, "shape": [ 3584, 18944 ], "size": 46678016, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 671148032, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 709339136, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 709353472, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 709355520, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 710387712, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 717613056, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 717627392, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.2.attn_v.bias", "offset": 724852736, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 724854784, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 726116352, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 726130688, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 764321792, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 802512896, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 840704000, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 840718336, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 840720384, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 841752576, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 848977920, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 848992256, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.3.attn_v.bias", "offset": 856217600, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 856219648, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 857481216, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 857495552, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 895686656, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 933877760, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 972068864, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 972083200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 972085248, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 973117440, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 980342784, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 980357120, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.4.attn_v.bias", "offset": 987582464, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 987584512, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 988616704, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 988631040, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 1026822144, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1065013248, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1103204352, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1103218688, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1103220736, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 1104252928, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 1111478272, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1111492608, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.5.attn_v.bias", "offset": 1118717952, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1118720000, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_k.bias", "offset": 1119752192, "shape": [ 512 ], "size": 1032192, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1119754240, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 1120786432, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 1128011776, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1128026112, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.attn_v.bias", "offset": 1135251456, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1135253504, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1136285696, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1136300032, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1174491136, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1212682240, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1250873344, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1250887680, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1250889728, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 1251921920, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 1259147264, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1259161600, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.attn_v.bias", "offset": 1266386944, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1266388992, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1267421184, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1267435520, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1305626624, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1343817728, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1382008832, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1382023168, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1382025216, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 1383057408, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 1390282752, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1390297088, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.attn_v.bias", "offset": 1397522432, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1397524480, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1398556672, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1398571008, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 1436762112, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1474953216, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1513144320, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1513158656, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1513160704, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 1514192896, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 1521418240, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1521432576, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.12.attn_v.bias", "offset": 1528657920, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1528659968, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1529692160, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1529706496, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1567897600, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1606088704, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 1644279808, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1644294144, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1644296192, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1645328384, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 1652553728, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1652568064, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.13.attn_v.bias", "offset": 1659793408, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1659795456, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1660827648, "shape": [ 3584, 18944 ], "size": 1032192, "type": 12 }, { "name": "blk.14.attn_k.bias", "offset": 1699018752, "shape": [ 512 ], "size": 38191104, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1699020800, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 1700052992, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 1707278336, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1707292672, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.attn_v.bias", "offset": 1714518016, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1714520064, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 1715552256, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1715566592, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 1753757696, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1791948800, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 1830139904, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1830154240, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1830168576, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 1868359680, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 1906550784, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 1944741888, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1944756224, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1944758272, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 1945790464, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 1953015808, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1953030144, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.7.attn_v.bias", "offset": 1960255488, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1960257536, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1961289728, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1961304064, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 1999495168, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2037686272, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2075877376, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2075891712, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2075893760, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 2076925952, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 2084151296, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2084165632, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.attn_v.bias", "offset": 2091390976, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2091393024, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2092425216, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2092439552, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 2130630656, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2168821760, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2207012864, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2207027200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2207029248, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 2208061440, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 2215286784, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2215301120, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.9.attn_v.bias", "offset": 2222526464, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2222528512, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2223560704, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2223575040, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 2261766144, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 2299957248, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 2299971584, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2299985920, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 2338177024, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 2376368128, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 2414559232, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2414573568, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2414575616, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 2415607808, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 2422833152, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2422847488, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.attn_v.bias", "offset": 2430072832, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2430074880, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2431107072, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2431121408, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 2469312512, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 2507503616, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 2545694720, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 2545709056, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2545711104, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 2546743296, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 2553968640, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2553982976, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.16.attn_v.bias", "offset": 2561208320, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 2561210368, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2562242560, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2562256896, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 2600448000, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 2638639104, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 2676830208, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 2676844544, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2676846592, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 2677878784, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 2685104128, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2685118464, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.17.attn_v.bias", "offset": 2692343808, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 2692345856, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 2693378048, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2693392384, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2731583488, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 2769774592, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 2807965696, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 2807980032, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2807982080, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 2809014272, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 2816239616, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2816253952, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.attn_v.bias", "offset": 2823479296, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 2823481344, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 2824513536, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2824527872, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 2862718976, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 2900910080, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 2939101184, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 2939115520, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2939117568, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 2940149760, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 2947375104, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2947389440, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.19.attn_v.bias", "offset": 2954614784, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 2954616832, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 2955649024, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2955663360, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 2993854464, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 3032045568, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 3070236672, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 3070251008, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3070253056, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 3071285248, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 3078510592, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3078524928, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.20.attn_v.bias", "offset": 3085750272, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 3085752320, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 3086784512, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3086798848, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 3124989952, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 3163181056, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 3201372160, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 3201386496, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3201388544, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 3202420736, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 3209646080, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3209660416, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.attn_v.bias", "offset": 3216885760, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 3216887808, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 3217920000, "shape": [ 3584, 18944 ], "size": 1032192, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 3256111104, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.22.attn_k.bias", "offset": 3294302208, "shape": [ 512 ], "size": 38191104, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3294304256, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 3295336448, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 3302561792, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 3302576128, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.22.attn_v.bias", "offset": 3309801472, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 3309803520, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "output.weight", "offset": 3310835712, "shape": [ 3584, 152064 ], "size": 1032192, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 3757903872, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3757918208, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 3796109312, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 3796123648, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3796137984, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 3834329088, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 3872520192, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 3910711296, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 3910725632, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3910727680, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 3911759872, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 3918985216, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 3918999552, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.23.attn_v.bias", "offset": 3926224896, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 3926226944, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 3927259136, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3927273472, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 3965464576, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 4003655680, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 4041846784, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 4041861120, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4041863168, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 4042895360, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 4050120704, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 4050135040, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.24.attn_v.bias", "offset": 4057360384, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 4057362432, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4058394624, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4058408960, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 4096600064, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 4134791168, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 4172982272, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 4172996608, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4172998656, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 4174030848, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 4181256192, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 4181270528, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.25.attn_v.bias", "offset": 4188495872, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 4188497920, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 4189530112, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4189544448, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 4227735552, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 4265926656, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 4304117760, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 4304132096, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4304134144, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 4305166336, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 4312391680, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 4312406016, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.26.attn_v.bias", "offset": 4319631360, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 4319633408, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 4320665600, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4320679936, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 4358871040, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 4397062144, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 4435253248, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 4435267584, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4435269632, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 4436301824, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 4443527168, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 4443541504, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.27.attn_v.bias", "offset": 4450766848, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 4450768896, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "output_norm.weight", "offset": 4451801088, "shape": [ 3584 ], "size": 1032192, "type": 0 } ], "version": 3 }