qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 15, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 306561024, "shape": [ 3584 ], "size": 306561024, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 306575360, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 362270720, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 400461824, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 438652928, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 438667264, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 438669312, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 439701504, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 446926848, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 446941184, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.0.attn_v.bias", "offset": 454166528, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 454168576, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 455673856, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 455688192, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 511383552, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 549574656, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 587765760, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 587780096, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 587782144, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 588814336, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 596039680, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 596054016, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.1.attn_v.bias", "offset": 603279360, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 603281408, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 604786688, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 604801024, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 660496384, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 698687488, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 736878592, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 736892928, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 736894976, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 737927168, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 745152512, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 745166848, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.2.attn_v.bias", "offset": 752392192, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 752394240, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 753899520, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 753913856, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 792104960, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 830296064, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 868487168, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 868501504, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 868503552, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 869535744, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 876761088, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 876775424, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.3.attn_v.bias", "offset": 884000768, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 884002816, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 885035008, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 885049344, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 923240448, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 961431552, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 999622656, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 999636992, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 999639040, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 1000671232, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 1007896576, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1007910912, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.4.attn_v.bias", "offset": 1015136256, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000\u0000blk.16.attn_v.", "offset": 1015138304, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1016170496, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1016184832, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 1071880192, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1110071296, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1148262400, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1148276736, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1148278784, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 1149310976, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 1156536320, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1156550656, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.5.attn_v.bias", "offset": 1163776000, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1163778048, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_k.bias", "offset": 1165283328, "shape": [ 512 ], "size": 1505280, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1165285376, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 1166317568, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 1173542912, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1173557248, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.attn_v.bias", "offset": 1180782592, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1180784640, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1181816832, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1181831168, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 1220022272, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1258213376, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1296404480, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1296418816, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1296420864, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 1297453056, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 1304678400, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1304692736, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.10.attn_v.bias", "offset": 1311918080, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1311920128, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1312952320, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1312966656, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 1351157760, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1389348864, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1427539968, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1427554304, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1427556352, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 1428588544, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 1435813888, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1435828224, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.11.attn_v.bias", "offset": 1443053568, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1443055616, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 1444560896, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1444575232, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 1500270592, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1538461696, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1576652800, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1576667136, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1576669184, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 1577701376, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 1584926720, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1584941056, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.12.attn_v.bias", "offset": 1592166400, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1592168448, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1593200640, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1593214976, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 1631406080, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1669597184, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 1707788288, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1707802624, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1707804672, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 1708836864, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 1716062208, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1716076544, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.13.attn_v.bias", "offset": 1723301888, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1723303936, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1724336128, "shape": [ 3584, 18944 ], "size": 1032192, "type": 12 }, { "name": "blk.14.attn_k.bias", "offset": 1762527232, "shape": [ 512 ], "size": 38191104, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1762529280, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 1763561472, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 1770786816, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1770801152, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.14.attn_v.bias", "offset": 1778026496, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1778028544, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 1779533824, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1779548160, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 1817739264, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1855930368, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 1894121472, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1894135808, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1894150144, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 1949845504, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 1988036608, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2026227712, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2026242048, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2026244096, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 2027276288, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 2034501632, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2034515968, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.7.attn_v.bias", "offset": 2041741312, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2041743360, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2042775552, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2042789888, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 2080980992, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2119172096, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2157363200, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2157377536, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2157379584, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 2158411776, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 2165637120, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2165651456, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.8.attn_v.bias", "offset": 2172876800, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2172878848, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2173911040, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2173925376, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 2212116480, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2250307584, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2288498688, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2288513024, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2288515072, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 2289547264, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 2296772608, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2296786944, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.9.attn_v.bias", "offset": 2304012288, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2304014336, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 2305519616, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2305533952, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 2361229312, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 2399420416, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 2399434752, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2399449088, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 2437640192, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 2475831296, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 2514022400, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2514036736, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2514038784, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 2515070976, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 2522296320, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2522310656, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.15.attn_v.bias", "offset": 2529536000, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2529538048, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2530570240, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2530584576, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 2568775680, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 2606966784, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 2645157888, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 2645172224, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2645174272, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 2646206464, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 2653431808, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2653446144, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.16.attn_v.bias", "offset": 2660671488, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "\ufffd\r\ufffd\ufffds\ufffd\ufffd\ufffd\ufffd\ufffd\ufffd5\ufffd\ufffd\ufffd8\ufffdWVj", "offset": 2660673536, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2661705728, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2661720064, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 2717415424, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 2755606528, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 2793797632, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 2793811968, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2793814016, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 2794846208, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 2802071552, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2802085888, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.17.attn_v.bias", "offset": 2809311232, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 2809313280, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 2810818560, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2810832896, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 2849024000, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 2887215104, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 2925406208, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 2925420544, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2925422592, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 2926454784, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 2933680128, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2933694464, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.18.attn_v.bias", "offset": 2940919808, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 2940921856, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 2941954048, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2941968384, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 2980159488, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 3018350592, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 3056541696, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 3056556032, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3056558080, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 3057590272, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 3064815616, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 3064829952, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.19.attn_v.bias", "offset": 3072055296, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 3072057344, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3073089536, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3073103872, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 3128799232, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 3166990336, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 3205181440, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 3205195776, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3205197824, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 3206230016, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 3213455360, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3213469696, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.20.attn_v.bias", "offset": 3220695040, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 3220697088, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 3222202368, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3222216704, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 3260407808, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 3298598912, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 3336790016, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 3336804352, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3336806400, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 3337838592, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 3345063936, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3345078272, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.21.attn_v.bias", "offset": 3352303616, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 3352305664, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 3353337856, "shape": [ 3584, 18944 ], "size": 1032192, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 3391528960, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.22.attn_k.bias", "offset": 3429720064, "shape": [ 512 ], "size": 38191104, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3429722112, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 3430754304, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 3437979648, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 3437993984, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.22.attn_v.bias", "offset": 3445219328, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 3445221376, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "output.weight", "offset": 3446253568, "shape": [ 3584, 152064 ], "size": 1032192, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 3893321728, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3893336064, "shape": [ 18944, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 3931527168, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 3931541504, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3931555840, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 3987251200, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 4025442304, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 4063633408, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 4063647744, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4063649792, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 4064681984, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 4071907328, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 4071921664, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.23.attn_v.bias", "offset": 4079147008, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 4079149056, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 4080654336, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4080668672, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 4136364032, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 4174555136, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 4212746240, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 4212760576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4212762624, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 4213794816, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 4221020160, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 4221034496, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.24.attn_v.bias", "offset": 4228259840, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 4228261888, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 4229767168, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4229781504, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 4285476864, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 4323667968, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 4361859072, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 4361873408, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4361875456, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 4362907648, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 4370132992, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 4370147328, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.25.attn_v.bias", "offset": 4377372672, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 4377374720, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 4378880000, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4378894336, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 4434589696, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 4472780800, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 4510971904, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 4510986240, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4510988288, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 4512020480, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 4519245824, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 4519260160, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.26.attn_v.bias", "offset": 4526485504, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 4526487552, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 4527992832, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4528007168, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 4583702528, "shape": [ 3584, 18944 ], "size": 55695360, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 4621893632, "shape": [ 3584, 18944 ], "size": 38191104, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 4660084736, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 4660099072, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4660101120, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 4661133312, "shape": [ 3584, 3584 ], "size": 1032192, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 4668358656, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 4668372992, "shape": [ 3584, 3584 ], "size": 14336, "type": 12 }, { "name": "blk.27.attn_v.bias", "offset": 4675598336, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 4675600384, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "output_norm.weight", "offset": 4677105664, "shape": [ 3584 ], "size": 1505280, "type": 0 } ], "version": 3 }