qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 2, "general.name": "Qwen2-7B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 306561024, "shape": [ 3584 ], "size": 306561024, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 306575360, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 344766464, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 382957568, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 421148672, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 421163008, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 421165056, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 422197248, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.0.attn_q.bias", "offset": 429422592, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 429436928, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.0.attn_v.bias", "offset": 436662272, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 436664320, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 437696512, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 437710848, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 475901952, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 514093056, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 552284160, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 552298496, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 552300544, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 553332736, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.1.attn_q.bias", "offset": 560558080, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 560572416, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.1.attn_v.bias", "offset": 567797760, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 567799808, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 568832000, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 568846336, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 607037440, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 645228544, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 683419648, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 683433984, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 683436032, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 684468224, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.2.attn_q.bias", "offset": 691693568, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 691707904, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.2.attn_v.bias", "offset": 698933248, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 698935296, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 699967488, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 699981824, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 738172928, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 776364032, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 814555136, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 814569472, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 814571520, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 815603712, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.3.attn_q.bias", "offset": 822829056, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 822843392, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.3.attn_v.bias", "offset": 830068736, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 830070784, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 831102976, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 831117312, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 869308416, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 907499520, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 945690624, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 945704960, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 945707008, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 946739200, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.4.attn_q.bias", "offset": 953964544, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 953978880, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.4.attn_v.bias", "offset": 961204224, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 961206272, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 962238464, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 962252800, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 1000443904, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 1038635008, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1076826112, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1076840448, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1076842496, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 1077874688, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.5.attn_q.bias", "offset": 1085100032, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1085114368, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.5.attn_v.bias", "offset": 1092339712, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1092341760, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.6.attn_k.bias", "offset": 1093373952, "shape": [ 512 ], "size": 1032192, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1093376000, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1094408192, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.6.attn_q.bias", "offset": 1101633536, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1101647872, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.6.attn_v.bias", "offset": 1108873216, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1108875264, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1109907456, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1109921792, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 1148112896, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 1186304000, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1224495104, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1224509440, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1224511488, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 1225543680, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.10.attn_q.bias", "offset": 1232769024, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1232783360, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.10.attn_v.bias", "offset": 1240008704, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1240010752, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 1241042944, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1241057280, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 1279248384, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1317439488, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 1355630592, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1355644928, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1355646976, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 1356679168, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.11.attn_q.bias", "offset": 1363904512, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1363918848, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.11.attn_v.bias", "offset": 1371144192, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1371146240, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1372178432, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1372192768, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 1410383872, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 1448574976, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 1486766080, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1486780416, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1486782464, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 1487814656, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.12.attn_q.bias", "offset": 1495040000, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1495054336, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.12.attn_v.bias", "offset": 1502279680, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1502281728, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 1503313920, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1503328256, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 1541519360, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 1579710464, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 1617901568, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1617915904, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1617917952, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 1618950144, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.13.attn_q.bias", "offset": 1626175488, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1626189824, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.13.attn_v.bias", "offset": 1633415168, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1633417216, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 1634449408, "shape": [ 3584, 18944 ], "size": 1032192, "type": 2 }, { "name": "blk.14.attn_k.bias", "offset": 1672640512, "shape": [ 512 ], "size": 38191104, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1672642560, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 1673674752, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.14.attn_q.bias", "offset": 1680900096, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1680914432, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.14.attn_v.bias", "offset": 1688139776, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1688141824, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 1689174016, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1689188352, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 1727379456, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 1765570560, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 1803761664, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1803776000, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1803790336, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 1841981440, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 1880172544, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 1918363648, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1918377984, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1918380032, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 1919412224, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.7.attn_q.bias", "offset": 1926637568, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1926651904, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.7.attn_v.bias", "offset": 1933877248, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1933879296, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 1934911488, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1934925824, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 1973116928, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 2011308032, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 2049499136, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2049513472, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2049515520, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 2050547712, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.8.attn_q.bias", "offset": 2057773056, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2057787392, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.8.attn_v.bias", "offset": 2065012736, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2065014784, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 2066046976, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2066061312, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 2104252416, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 2142443520, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 2180634624, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2180648960, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2180651008, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 2181683200, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.9.attn_q.bias", "offset": 2188908544, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2188922880, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.9.attn_v.bias", "offset": 2196148224, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2196150272, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 2197182464, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2197196800, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 2235387904, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 2273579008, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 2273593344, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2273607680, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 2311798784, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 2349989888, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 2388180992, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2388195328, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2388197376, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 2389229568, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.15.attn_q.bias", "offset": 2396454912, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2396469248, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.15.attn_v.bias", "offset": 2403694592, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2403696640, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 2404728832, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2404743168, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 2442934272, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 2481125376, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 2519316480, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 2519330816, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2519332864, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 2520365056, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.16.attn_q.bias", "offset": 2527590400, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2527604736, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.16.attn_v.bias", "offset": 2534830080, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 2534832128, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 2535864320, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2535878656, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 2574069760, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 2612260864, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 2650451968, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 2650466304, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2650468352, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 2651500544, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.17.attn_q.bias", "offset": 2658725888, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2658740224, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.17.attn_v.bias", "offset": 2665965568, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 2665967616, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 2666999808, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2667014144, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 2705205248, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 2743396352, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 2781587456, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 2781601792, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2781603840, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 2782636032, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.18.attn_q.bias", "offset": 2789861376, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2789875712, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.18.attn_v.bias", "offset": 2797101056, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 2797103104, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 2798135296, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2798149632, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 2836340736, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 2874531840, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 2912722944, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 2912737280, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2912739328, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 2913771520, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.19.attn_q.bias", "offset": 2920996864, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2921011200, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.19.attn_v.bias", "offset": 2928236544, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 2928238592, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 2929270784, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2929285120, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 2967476224, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 3005667328, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 3043858432, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 3043872768, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3043874816, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 3044907008, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.20.attn_q.bias", "offset": 3052132352, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3052146688, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.20.attn_v.bias", "offset": 3059372032, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 3059374080, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 3060406272, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3060420608, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 3098611712, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 3136802816, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 3174993920, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 3175008256, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3175010304, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 3176042496, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.21.attn_q.bias", "offset": 3183267840, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3183282176, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.21.attn_v.bias", "offset": 3190507520, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 3190509568, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 3191541760, "shape": [ 3584, 18944 ], "size": 1032192, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 3229732864, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.22.attn_k.bias", "offset": 3267923968, "shape": [ 512 ], "size": 38191104, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3267926016, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 3268958208, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.22.attn_q.bias", "offset": 3276183552, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 3276197888, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.22.attn_v.bias", "offset": 3283423232, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 3283425280, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "output.weight", "offset": 3284457472, "shape": [ 3584, 152064 ], "size": 1032192, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 3731525632, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3731539968, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 3769731072, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 3769745408, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3769759744, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 3807950848, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 3846141952, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 3884333056, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 3884347392, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3884349440, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 3885381632, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.23.attn_q.bias", "offset": 3892606976, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 3892621312, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.23.attn_v.bias", "offset": 3899846656, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 3899848704, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 3900880896, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3900895232, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 3939086336, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 3977277440, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 4015468544, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 4015482880, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4015484928, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 4016517120, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.24.attn_q.bias", "offset": 4023742464, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 4023756800, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.24.attn_v.bias", "offset": 4030982144, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 4030984192, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 4032016384, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4032030720, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 4070221824, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 4108412928, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 4146604032, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 4146618368, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4146620416, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 4147652608, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.25.attn_q.bias", "offset": 4154877952, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 4154892288, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.25.attn_v.bias", "offset": 4162117632, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 4162119680, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 4163151872, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4163166208, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 4201357312, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 4239548416, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 4277739520, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 4277753856, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4277755904, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 4278788096, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.26.attn_q.bias", "offset": 4286013440, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 4286027776, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.26.attn_v.bias", "offset": 4293253120, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 4293255168, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 4294287360, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4294301696, "shape": [ 18944, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 4332492800, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 4370683904, "shape": [ 3584, 18944 ], "size": 38191104, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 4408875008, "shape": [ 3584 ], "size": 38191104, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 4408889344, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4408891392, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 4409923584, "shape": [ 3584, 3584 ], "size": 1032192, "type": 2 }, { "name": "blk.27.attn_q.bias", "offset": 4417148928, "shape": [ 3584 ], "size": 7225344, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 4417163264, "shape": [ 3584, 3584 ], "size": 14336, "type": 2 }, { "name": "blk.27.attn_v.bias", "offset": 4424388608, "shape": [ 512 ], "size": 7225344, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 4424390656, "shape": [ 3584, 512 ], "size": 2048, "type": 2 }, { "name": "output_norm.weight", "offset": 4425422848, "shape": [ 3584 ], "size": 1032192, "type": 0 } ], "version": 3 }