qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 8, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 374685696, "shape": [ 3584 ], "size": 374685696, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 374700032, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.0.ffn_gate.weight", "offset": 421378048, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 468056064, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 514734080, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 514748416, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 514750464, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 516012032, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.0.attn_q.bias", "offset": 524843008, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 524857344, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.0.attn_v.bias", "offset": 533688320, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 533690368, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 534951936, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 534966272, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.1.ffn_gate.weight", "offset": 581644288, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 628322304, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 675000320, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 675014656, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 675016704, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 676278272, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.1.attn_q.bias", "offset": 685109248, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 685123584, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.1.attn_v.bias", "offset": 693954560, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 693956608, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 695218176, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 695232512, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.2.ffn_gate.weight", "offset": 741910528, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 788588544, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 835266560, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 835280896, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 835282944, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 836544512, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.2.attn_q.bias", "offset": 845375488, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 845389824, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.2.attn_v.bias", "offset": 854220800, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 854222848, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 855484416, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 855498752, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.3.ffn_gate.weight", "offset": 902176768, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 948854784, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 995532800, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 995547136, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 995549184, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 996810752, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.3.attn_q.bias", "offset": 1005641728, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1005656064, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.3.attn_v.bias", "offset": 1014487040, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1014489088, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 1015750656, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1015764992, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.4.ffn_gate.weight", "offset": 1062443008, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 1109121024, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 1155799040, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1155813376, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1155815424, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 1157076992, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.4.attn_q.bias", "offset": 1165907968, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1165922304, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.4.attn_v.bias", "offset": 1174753280, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1174755328, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 1176016896, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1176031232, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.5.ffn_gate.weight", "offset": 1222709248, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 1269387264, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 1316065280, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1316079616, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1316081664, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 1317343232, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.5.attn_q.bias", "offset": 1326174208, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1326188544, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.5.attn_v.bias", "offset": 1335019520, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1335021568, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.6.attn_k.bias", "offset": 1336283136, "shape": [ 512 ], "size": 1261568, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1336285184, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 1337546752, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.6.attn_q.bias", "offset": 1346377728, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1346392064, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.6.attn_v.bias", "offset": 1355223040, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1355225088, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 1356486656, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1356500992, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.10.ffn_gate.weight", "offset": 1403179008, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 1449857024, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 1496535040, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1496549376, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1496551424, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 1497812992, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.10.attn_q.bias", "offset": 1506643968, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1506658304, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.10.attn_v.bias", "offset": 1515489280, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1515491328, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 1516752896, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1516767232, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.11.ffn_gate.weight", "offset": 1563445248, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 1610123264, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 1656801280, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1656815616, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1656817664, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 1658079232, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.11.attn_q.bias", "offset": 1666910208, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1666924544, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.11.attn_v.bias", "offset": 1675755520, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1675757568, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 1677019136, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1677033472, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.12.ffn_gate.weight", "offset": 1723711488, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 1770389504, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 1817067520, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1817081856, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1817083904, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 1818345472, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.12.attn_q.bias", "offset": 1827176448, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1827190784, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.12.attn_v.bias", "offset": 1836021760, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1836023808, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 1837285376, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1837299712, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.13.ffn_gate.weight", "offset": 1883977728, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 1930655744, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 1977333760, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1977348096, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1977350144, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 1978611712, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.13.attn_q.bias", "offset": 1987442688, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1987457024, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.13.attn_v.bias", "offset": 1996288000, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1996290048, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.14.ffn_gate.weight", "offset": 1997551616, "shape": [ 3584, 18944 ], "size": 1261568, "type": 6 }, { "name": "blk.14.attn_k.bias", "offset": 2044229632, "shape": [ 512 ], "size": 46678016, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2044231680, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 2045493248, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.14.attn_q.bias", "offset": 2054324224, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 2054338560, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.14.attn_v.bias", "offset": 2063169536, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 2063171584, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 2064433152, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2064447488, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.6.ffn_gate.weight", "offset": 2111125504, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 2157803520, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 2204481536, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2204495872, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2204510208, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.7.ffn_gate.weight", "offset": 2251188224, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 2297866240, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 2344544256, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2344558592, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2344560640, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 2345822208, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.7.attn_q.bias", "offset": 2354653184, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2354667520, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.7.attn_v.bias", "offset": 2363498496, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2363500544, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 2364762112, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2364776448, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.8.ffn_gate.weight", "offset": 2411454464, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 2458132480, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 2504810496, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2504824832, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2504826880, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 2506088448, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.8.attn_q.bias", "offset": 2514919424, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2514933760, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.8.attn_v.bias", "offset": 2523764736, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2523766784, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 2525028352, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2525042688, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.9.ffn_gate.weight", "offset": 2571720704, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 2618398720, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 2665076736, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2665091072, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2665093120, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 2666354688, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.9.attn_q.bias", "offset": 2675185664, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2675200000, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.9.attn_v.bias", "offset": 2684030976, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2684033024, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 2685294592, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2685308928, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 2731986944, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 2778664960, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 2778679296, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2778693632, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.15.ffn_gate.weight", "offset": 2825371648, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 2872049664, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 2918727680, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2918742016, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2918744064, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 2920005632, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.15.attn_q.bias", "offset": 2928836608, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2928850944, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.15.attn_v.bias", "offset": 2937681920, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2937683968, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 2938945536, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2938959872, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.16.ffn_gate.weight", "offset": 2985637888, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 3032315904, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 3078993920, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 3079008256, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3079010304, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 3080271872, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.16.attn_q.bias", "offset": 3089102848, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 3089117184, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.16.attn_v.bias", "offset": 3097948160, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 3097950208, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 3099211776, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3099226112, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.17.ffn_gate.weight", "offset": 3145904128, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 3192582144, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 3239260160, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 3239274496, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3239276544, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 3240538112, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.17.attn_q.bias", "offset": 3249369088, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 3249383424, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.17.attn_v.bias", "offset": 3258214400, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 3258216448, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 3259478016, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3259492352, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.18.ffn_gate.weight", "offset": 3306170368, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 3352848384, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 3399526400, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 3399540736, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3399542784, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 3400804352, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.18.attn_q.bias", "offset": 3409635328, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 3409649664, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.18.attn_v.bias", "offset": 3418480640, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 3418482688, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 3419744256, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3419758592, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.19.ffn_gate.weight", "offset": 3466436608, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 3513114624, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 3559792640, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 3559806976, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3559809024, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 3561070592, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.19.attn_q.bias", "offset": 3569901568, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 3569915904, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.19.attn_v.bias", "offset": 3578746880, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 3578748928, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 3580010496, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3580024832, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.20.ffn_gate.weight", "offset": 3626702848, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 3673380864, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 3720058880, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 3720073216, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3720075264, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 3721336832, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.20.attn_q.bias", "offset": 3730167808, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3730182144, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.20.attn_v.bias", "offset": 3739013120, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 3739015168, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 3740276736, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3740291072, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.21.ffn_gate.weight", "offset": 3786969088, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 3833647104, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 3880325120, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 3880339456, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3880341504, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 3881603072, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.21.attn_q.bias", "offset": 3890434048, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3890448384, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.21.attn_v.bias", "offset": 3899279360, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 3899281408, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.22.ffn_gate.weight", "offset": 3900542976, "shape": [ 3584, 18944 ], "size": 1261568, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 3947220992, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.22.attn_k.bias", "offset": 3993899008, "shape": [ 512 ], "size": 46678016, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3993901056, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 3995162624, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.22.attn_q.bias", "offset": 4003993600, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 4004007936, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.22.attn_v.bias", "offset": 4012838912, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 4012840960, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "output.weight", "offset": 4014102528, "shape": [ 3584, 152064 ], "size": 1261568, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 4461170688, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4461185024, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 4507863040, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 4507877376, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4507891712, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.23.ffn_gate.weight", "offset": 4554569728, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 4601247744, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 4647925760, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 4647940096, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4647942144, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 4649203712, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.23.attn_q.bias", "offset": 4658034688, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 4658049024, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.23.attn_v.bias", "offset": 4666880000, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 4666882048, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.24.attn_norm.weight", "offset": 4668143616, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4668157952, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.24.ffn_gate.weight", "offset": 4714835968, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.24.ffn_up.weight", "offset": 4761513984, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.24.ffn_norm.weight", "offset": 4808192000, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 4808206336, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4808208384, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.24.attn_output.weight", "offset": 4809469952, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.24.attn_q.bias", "offset": 4818300928, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 4818315264, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.24.attn_v.bias", "offset": 4827146240, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 4827148288, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.25.attn_norm.weight", "offset": 4828409856, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4828424192, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.25.ffn_gate.weight", "offset": 4875102208, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 4921780224, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.25.ffn_norm.weight", "offset": 4968458240, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 4968472576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4968474624, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.25.attn_output.weight", "offset": 4969736192, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.25.attn_q.bias", "offset": 4978567168, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 4978581504, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.25.attn_v.bias", "offset": 4987412480, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 4987414528, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.26.attn_norm.weight", "offset": 4988676096, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4988690432, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.26.ffn_gate.weight", "offset": 5035368448, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.26.ffn_up.weight", "offset": 5082046464, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.26.ffn_norm.weight", "offset": 5128724480, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 5128738816, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5128740864, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.26.attn_output.weight", "offset": 5130002432, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.26.attn_q.bias", "offset": 5138833408, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 5138847744, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.26.attn_v.bias", "offset": 5147678720, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 5147680768, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.27.attn_norm.weight", "offset": 5148942336, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5148956672, "shape": [ 18944, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.27.ffn_gate.weight", "offset": 5195634688, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.27.ffn_up.weight", "offset": 5242312704, "shape": [ 3584, 18944 ], "size": 46678016, "type": 6 }, { "name": "blk.27.ffn_norm.weight", "offset": 5288990720, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 5289005056, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5289007104, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "blk.27.attn_output.weight", "offset": 5290268672, "shape": [ 3584, 3584 ], "size": 1261568, "type": 6 }, { "name": "blk.27.attn_q.bias", "offset": 5299099648, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 5299113984, "shape": [ 3584, 3584 ], "size": 14336, "type": 6 }, { "name": "blk.27.attn_v.bias", "offset": 5307944960, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 5307947008, "shape": [ 3584, 512 ], "size": 2048, "type": 6 }, { "name": "output_norm.weight", "offset": 5309208576, "shape": [ 3584 ], "size": 1261568, "type": 0 } ], "version": 3 }