qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 17, "general.name": "Qwen2-7B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 374685696, "shape": [ 3584 ], "size": 374685696, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 374700032, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 430395392, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 477073408, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 523751424, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 523765760, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 523767808, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 525029376, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 533860352, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 533874688, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.0.attn_v.bias", "offset": 542705664, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 542707712, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 544212992, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 544227328, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 599922688, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 646600704, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 693278720, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 693293056, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 693295104, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 694556672, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 703387648, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 703401984, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.1.attn_v.bias", "offset": 712232960, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 712235008, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 713740288, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 713754624, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 769449984, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 816128000, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 862806016, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 862820352, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 862822400, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 864083968, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 872914944, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 872929280, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.2.attn_v.bias", "offset": 881760256, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 881762304, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 883267584, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 883281920, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 929959936, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 976637952, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 1023315968, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1023330304, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1023332352, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 1024593920, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 1033424896, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1033439232, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.3.attn_v.bias", "offset": 1042270208, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1042272256, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1043533824, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1043548160, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 1090226176, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 1136904192, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 1183582208, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1183596544, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1183598592, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 1184860160, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 1193691136, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1193705472, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.4.attn_v.bias", "offset": 1202536448, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1202538496, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1203800064, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1203814400, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 1259509760, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1306187776, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 1352865792, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1352880128, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1352882176, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 1354143744, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 1362974720, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1362989056, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.5.attn_v.bias", "offset": 1371820032, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1371822080, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_k.bias", "offset": 1373327360, "shape": [ 512 ], "size": 1505280, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1373329408, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 1374590976, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 1383421952, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1383436288, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.6.attn_v.bias", "offset": 1392267264, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1392269312, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1393530880, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1393545216, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 1440223232, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 1486901248, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 1533579264, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1533593600, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1533595648, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 1534857216, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 1543688192, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1543702528, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.10.attn_v.bias", "offset": 1552533504, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1552535552, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1553797120, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1553811456, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 1600489472, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 1647167488, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 1693845504, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1693859840, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1693861888, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 1695123456, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 1703954432, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1703968768, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.11.attn_v.bias", "offset": 1712799744, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1712801792, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 1714307072, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1714321408, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 1770016768, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 1816694784, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 1863372800, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1863387136, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1863389184, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 1864650752, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 1873481728, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1873496064, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.12.attn_v.bias", "offset": 1882327040, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1882329088, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 1883590656, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1883604992, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 1930283008, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 1976961024, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 2023639040, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 2023653376, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2023655424, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 2024916992, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 2033747968, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 2033762304, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.13.attn_v.bias", "offset": 2042593280, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 2042595328, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 2043856896, "shape": [ 3584, 18944 ], "size": 1261568, "type": 13 }, { "name": "blk.14.attn_k.bias", "offset": 2090534912, "shape": [ 512 ], "size": 46678016, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2090536960, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 2091798528, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 2100629504, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 2100643840, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.14.attn_v.bias", "offset": 2109474816, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 2109476864, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 2110982144, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2110996480, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 2157674496, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 2204352512, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 2251030528, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2251044864, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2251059200, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 2306754560, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 2353432576, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 2400110592, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2400124928, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2400126976, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 2401388544, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 2410219520, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2410233856, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.7.attn_v.bias", "offset": 2419064832, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2419066880, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 2420328448, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2420342784, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 2467020800, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 2513698816, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 2560376832, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2560391168, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2560393216, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 2561654784, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 2570485760, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2570500096, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.8.attn_v.bias", "offset": 2579331072, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2579333120, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 2580594688, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2580609024, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 2627287040, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 2673965056, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 2720643072, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2720657408, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2720659456, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 2721921024, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 2730752000, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2730766336, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.9.attn_v.bias", "offset": 2739597312, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2739599360, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 2741104640, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2741118976, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 2796814336, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 2843492352, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 2843506688, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2843521024, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 2890199040, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 2936877056, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 2983555072, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2983569408, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2983571456, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 2984833024, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 2993664000, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2993678336, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.15.attn_v.bias", "offset": 3002509312, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 3002511360, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 3003772928, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3003787264, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 3050465280, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 3097143296, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 3143821312, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 3143835648, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3143837696, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 3145099264, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 3153930240, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 3153944576, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.16.attn_v.bias", "offset": 3162775552, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 3162777600, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 3164039168, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3164053504, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 3219748864, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 3266426880, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 3313104896, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 3313119232, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3313121280, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 3314382848, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 3323213824, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 3323228160, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.17.attn_v.bias", "offset": 3332059136, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 3332061184, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 3333566464, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3333580800, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 3380258816, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 3426936832, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 3473614848, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 3473629184, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3473631232, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 3474892800, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 3483723776, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 3483738112, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.18.attn_v.bias", "offset": 3492569088, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 3492571136, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 3493832704, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3493847040, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 3540525056, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 3587203072, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 3633881088, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 3633895424, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3633897472, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 3635159040, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 3643990016, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 3644004352, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.19.attn_v.bias", "offset": 3652835328, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 3652837376, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 3654098944, "shape": [ 3584 ], "size": 1261568, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3654113280, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 3709808640, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 3756486656, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 3803164672, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 3803179008, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3803181056, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 3804442624, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 3813273600, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3813287936, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.20.attn_v.bias", "offset": 3822118912, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 3822120960, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 3823626240, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3823640576, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 3870318592, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 3916996608, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 3963674624, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 3963688960, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3963691008, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 3964952576, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 3973783552, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3973797888, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.21.attn_v.bias", "offset": 3982628864, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 3982630912, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 3983892480, "shape": [ 3584, 18944 ], "size": 1261568, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 4030570496, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.22.attn_k.bias", "offset": 4077248512, "shape": [ 512 ], "size": 46678016, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4077250560, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 4078512128, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 4087343104, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 4087357440, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.22.attn_v.bias", "offset": 4096188416, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 4096190464, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "output.weight", "offset": 4097452032, "shape": [ 3584, 152064 ], "size": 1261568, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 4544520192, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4544534528, "shape": [ 18944, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 4591212544, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 4591226880, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4591241216, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 4646936576, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 4693614592, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 4740292608, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 4740306944, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4740308992, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 4741570560, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 4750401536, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 4750415872, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.23.attn_v.bias", "offset": 4759246848, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 4759248896, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 4760754176, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4760768512, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 4816463872, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 4863141888, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 4909819904, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 4909834240, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4909836288, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 4911097856, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 4919928832, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 4919943168, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.24.attn_v.bias", "offset": 4928774144, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 4928776192, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 4930281472, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4930295808, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 4985991168, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 5032669184, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 5079347200, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 5079361536, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5079363584, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 5080625152, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 5089456128, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 5089470464, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.25.attn_v.bias", "offset": 5098301440, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 5098303488, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 5099808768, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5099823104, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 5155518464, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 5202196480, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 5248874496, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 5248888832, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5248890880, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 5250152448, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 5258983424, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 5258997760, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.26.attn_v.bias", "offset": 5267828736, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 5267830784, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 5269336064, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5269350400, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 5325045760, "shape": [ 3584, 18944 ], "size": 55695360, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 5371723776, "shape": [ 3584, 18944 ], "size": 46678016, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 5418401792, "shape": [ 3584 ], "size": 46678016, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 5418416128, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5418418176, "shape": [ 3584, 512 ], "size": 2048, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 5419679744, "shape": [ 3584, 3584 ], "size": 1261568, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 5428510720, "shape": [ 3584 ], "size": 8830976, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 5428525056, "shape": [ 3584, 3584 ], "size": 14336, "type": 13 }, { "name": "blk.27.attn_v.bias", "offset": 5437356032, "shape": [ 512 ], "size": 8830976, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 5437358080, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "output_norm.weight", "offset": 5438863360, "shape": [ 3584 ], "size": 1505280, "type": 0 } ], "version": 3 }