qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 3, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 340623360, "shape": [ 3584 ], "size": 340623360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 340637696, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.0.ffn_gate.weight", "offset": 383072256, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 425506816, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 467941376, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 467955712, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 467957760, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.0.attn_output.weight", "offset": 469104640, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.0.attn_q.bias", "offset": 477132800, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 477147136, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.0.attn_v.bias", "offset": 485175296, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 485177344, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 486324224, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 486338560, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.1.ffn_gate.weight", "offset": 528773120, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 571207680, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 613642240, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 613656576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 613658624, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.1.attn_output.weight", "offset": 614805504, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.1.attn_q.bias", "offset": 622833664, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 622848000, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.1.attn_v.bias", "offset": 630876160, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 630878208, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 632025088, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 632039424, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.2.ffn_gate.weight", "offset": 674473984, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 716908544, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 759343104, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 759357440, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 759359488, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.2.attn_output.weight", "offset": 760506368, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.2.attn_q.bias", "offset": 768534528, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 768548864, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.2.attn_v.bias", "offset": 776577024, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 776579072, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 777725952, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 777740288, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.3.ffn_gate.weight", "offset": 820174848, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 862609408, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 905043968, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 905058304, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 905060352, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.3.attn_output.weight", "offset": 906207232, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.3.attn_q.bias", "offset": 914235392, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 914249728, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.3.attn_v.bias", "offset": 922277888, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 922279936, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 923426816, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 923441152, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.4.ffn_gate.weight", "offset": 965875712, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 1008310272, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 1050744832, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1050759168, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1050761216, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.4.attn_output.weight", "offset": 1051908096, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.4.attn_q.bias", "offset": 1059936256, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1059950592, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.4.attn_v.bias", "offset": 1067978752, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1067980800, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 1069127680, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1069142016, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.5.ffn_gate.weight", "offset": 1111576576, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 1154011136, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 1196445696, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1196460032, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1196462080, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.5.attn_output.weight", "offset": 1197608960, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.5.attn_q.bias", "offset": 1205637120, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1205651456, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.5.attn_v.bias", "offset": 1213679616, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1213681664, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.6.attn_k.bias", "offset": 1214828544, "shape": [ 512 ], "size": 1146880, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1214830592, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 1215977472, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.6.attn_q.bias", "offset": 1224005632, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1224019968, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.6.attn_v.bias", "offset": 1232048128, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1232050176, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 1233197056, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1233211392, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.10.ffn_gate.weight", "offset": 1275645952, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 1318080512, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 1360515072, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1360529408, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1360531456, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.10.attn_output.weight", "offset": 1361678336, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.10.attn_q.bias", "offset": 1369706496, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1369720832, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.10.attn_v.bias", "offset": 1377748992, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1377751040, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 1378897920, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1378912256, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.11.ffn_gate.weight", "offset": 1421346816, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 1463781376, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 1506215936, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1506230272, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1506232320, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.11.attn_output.weight", "offset": 1507379200, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.11.attn_q.bias", "offset": 1515407360, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1515421696, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.11.attn_v.bias", "offset": 1523449856, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1523451904, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 1524598784, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1524613120, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.12.ffn_gate.weight", "offset": 1567047680, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 1609482240, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 1651916800, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1651931136, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1651933184, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.12.attn_output.weight", "offset": 1653080064, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.12.attn_q.bias", "offset": 1661108224, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1661122560, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.12.attn_v.bias", "offset": 1669150720, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1669152768, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 1670299648, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1670313984, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.13.ffn_gate.weight", "offset": 1712748544, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 1755183104, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 1797617664, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1797632000, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1797634048, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.13.attn_output.weight", "offset": 1798780928, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.13.attn_q.bias", "offset": 1806809088, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1806823424, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.13.attn_v.bias", "offset": 1814851584, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1814853632, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.14.ffn_gate.weight", "offset": 1816000512, "shape": [ 3584, 18944 ], "size": 1146880, "type": 3 }, { "name": "blk.14.attn_k.bias", "offset": 1858435072, "shape": [ 512 ], "size": 42434560, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1858437120, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.14.attn_output.weight", "offset": 1859584000, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.14.attn_q.bias", "offset": 1867612160, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1867626496, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.14.attn_v.bias", "offset": 1875654656, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1875656704, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 1876803584, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1876817920, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.6.ffn_gate.weight", "offset": 1919252480, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 1961687040, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 2004121600, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2004135936, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2004150272, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.7.ffn_gate.weight", "offset": 2046584832, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 2089019392, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 2131453952, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2131468288, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2131470336, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.7.attn_output.weight", "offset": 2132617216, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.7.attn_q.bias", "offset": 2140645376, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2140659712, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.7.attn_v.bias", "offset": 2148687872, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2148689920, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 2149836800, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2149851136, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.8.ffn_gate.weight", "offset": 2192285696, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 2234720256, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 2277154816, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2277169152, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2277171200, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.8.attn_output.weight", "offset": 2278318080, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.8.attn_q.bias", "offset": 2286346240, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2286360576, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.8.attn_v.bias", "offset": 2294388736, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2294390784, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 2295537664, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2295552000, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.9.ffn_gate.weight", "offset": 2337986560, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 2380421120, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 2422855680, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2422870016, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2422872064, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.9.attn_output.weight", "offset": 2424018944, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.9.attn_q.bias", "offset": 2432047104, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2432061440, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.9.attn_v.bias", "offset": 2440089600, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2440091648, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 2441238528, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2441252864, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 2483687424, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 2526121984, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 2526136320, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2526150656, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.15.ffn_gate.weight", "offset": 2568585216, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 2611019776, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 2653454336, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 2653468672, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 2653470720, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.15.attn_output.weight", "offset": 2654617600, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.15.attn_q.bias", "offset": 2662645760, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 2662660096, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.15.attn_v.bias", "offset": 2670688256, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 2670690304, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 2671837184, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2671851520, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.16.ffn_gate.weight", "offset": 2714286080, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 2756720640, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 2799155200, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 2799169536, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 2799171584, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.16.attn_output.weight", "offset": 2800318464, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.16.attn_q.bias", "offset": 2808346624, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 2808360960, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.16.attn_v.bias", "offset": 2816389120, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 2816391168, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 2817538048, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2817552384, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.17.ffn_gate.weight", "offset": 2859986944, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 2902421504, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 2944856064, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 2944870400, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2944872448, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.17.attn_output.weight", "offset": 2946019328, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.17.attn_q.bias", "offset": 2954047488, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2954061824, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.17.attn_v.bias", "offset": 2962089984, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 2962092032, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 2963238912, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2963253248, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.18.ffn_gate.weight", "offset": 3005687808, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 3048122368, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 3090556928, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 3090571264, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3090573312, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.18.attn_output.weight", "offset": 3091720192, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.18.attn_q.bias", "offset": 3099748352, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 3099762688, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.18.attn_v.bias", "offset": 3107790848, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 3107792896, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 3108939776, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3108954112, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.19.ffn_gate.weight", "offset": 3151388672, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 3193823232, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 3236257792, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 3236272128, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3236274176, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.19.attn_output.weight", "offset": 3237421056, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.19.attn_q.bias", "offset": 3245449216, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 3245463552, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.19.attn_v.bias", "offset": 3253491712, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 3253493760, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 3254640640, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3254654976, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.20.ffn_gate.weight", "offset": 3297089536, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 3339524096, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 3381958656, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 3381972992, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 3381975040, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.20.attn_output.weight", "offset": 3383121920, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.20.attn_q.bias", "offset": 3391150080, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 3391164416, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.20.attn_v.bias", "offset": 3399192576, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 3399194624, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 3400341504, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3400355840, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.21.ffn_gate.weight", "offset": 3442790400, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 3485224960, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 3527659520, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 3527673856, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 3527675904, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 3528822784, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.21.attn_q.bias", "offset": 3536850944, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 3536865280, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.21.attn_v.bias", "offset": 3544893440, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 3544895488, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.22.ffn_gate.weight", "offset": 3546042368, "shape": [ 3584, 18944 ], "size": 1146880, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 3588476928, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.22.attn_k.bias", "offset": 3630911488, "shape": [ 512 ], "size": 42434560, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 3630913536, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.22.attn_output.weight", "offset": 3632060416, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.22.attn_q.bias", "offset": 3640088576, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 3640102912, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.22.attn_v.bias", "offset": 3648131072, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 3648133120, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "output.weight", "offset": 3649280000, "shape": [ 3584, 152064 ], "size": 1146880, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 4096348160, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4096362496, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 4138797056, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 4138811392, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4138825728, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.23.ffn_gate.weight", "offset": 4181260288, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 4223694848, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 4266129408, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 4266143744, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 4266145792, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.23.attn_output.weight", "offset": 4267292672, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.23.attn_q.bias", "offset": 4275320832, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 4275335168, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.23.attn_v.bias", "offset": 4283363328, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 4283365376, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 4284512256, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4284526592, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.24.ffn_gate.weight", "offset": 4326961152, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 4369395712, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 4411830272, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 4411844608, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 4411846656, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.24.attn_output.weight", "offset": 4412993536, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.24.attn_q.bias", "offset": 4421021696, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 4421036032, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.24.attn_v.bias", "offset": 4429064192, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 4429066240, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 4430213120, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4430227456, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.25.ffn_gate.weight", "offset": 4472662016, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 4515096576, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 4557531136, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 4557545472, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 4557547520, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.25.attn_output.weight", "offset": 4558694400, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.25.attn_q.bias", "offset": 4566722560, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 4566736896, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.25.attn_v.bias", "offset": 4574765056, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 4574767104, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 4575913984, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4575928320, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.26.ffn_gate.weight", "offset": 4618362880, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 4660797440, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 4703232000, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 4703246336, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 4703248384, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.26.attn_output.weight", "offset": 4704395264, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.26.attn_q.bias", "offset": 4712423424, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 4712437760, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.26.attn_v.bias", "offset": 4720465920, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 4720467968, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 4721614848, "shape": [ 3584 ], "size": 1146880, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4721629184, "shape": [ 18944, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.27.ffn_gate.weight", "offset": 4764063744, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 4806498304, "shape": [ 3584, 18944 ], "size": 42434560, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 4848932864, "shape": [ 3584 ], "size": 42434560, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 4848947200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 4848949248, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "blk.27.attn_output.weight", "offset": 4850096128, "shape": [ 3584, 3584 ], "size": 1146880, "type": 3 }, { "name": "blk.27.attn_q.bias", "offset": 4858124288, "shape": [ 3584 ], "size": 8028160, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 4858138624, "shape": [ 3584, 3584 ], "size": 14336, "type": 3 }, { "name": "blk.27.attn_v.bias", "offset": 4866166784, "shape": [ 512 ], "size": 8028160, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 4866168832, "shape": [ 3584, 512 ], "size": 2048, "type": 3 }, { "name": "output_norm.weight", "offset": 4867315712, "shape": [ 3584 ], "size": 1146880, "type": 0 } ], "version": 3 }