qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 9, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 408748032, "shape": [ 3584 ], "size": 408748032, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 408762368, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 459683840, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 510605312, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 561526784, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 561541120, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 561543168, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 562919424, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 572553216, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 572567552, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.0.attn_v.bias", "offset": 582201344, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 582203392, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 583579648, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 583593984, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 634515456, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 685436928, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 736358400, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 736372736, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 736374784, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 737751040, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 747384832, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 747399168, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.1.attn_v.bias", "offset": 757032960, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 757035008, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 758411264, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 758425600, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 809347072, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 860268544, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 911190016, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 911204352, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 911206400, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 912582656, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 922216448, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 922230784, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.2.attn_v.bias", "offset": 931864576, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 931866624, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 933242880, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 933257216, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 984178688, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 1035100160, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 1086021632, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1086035968, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1086038016, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 1087414272, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 1097048064, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1097062400, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.3.attn_v.bias", "offset": 1106696192, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1106698240, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 1108074496, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1108088832, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 1159010304, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 1209931776, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 1260853248, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1260867584, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1260869632, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 1262245888, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 1271879680, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1271894016, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.4.attn_v.bias", "offset": 1281527808, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1281529856, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 1282906112, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1282920448, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 1333841920, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 1384763392, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 1435684864, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1435699200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1435701248, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 1437077504, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 1446711296, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1446725632, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.5.attn_v.bias", "offset": 1456359424, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1456361472, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.6.attn_k.bias", "offset": 1457737728, "shape": [ 512 ], "size": 1376256, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1457739776, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1459116032, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 1468749824, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1468764160, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.6.attn_v.bias", "offset": 1478397952, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1478400000, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 1479776256, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1479790592, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 1530712064, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 1581633536, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 1632555008, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1632569344, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1632571392, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 1633947648, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 1643581440, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1643595776, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.10.attn_v.bias", "offset": 1653229568, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1653231616, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 1654607872, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1654622208, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.11.ffn_gate.weight", "offset": 1705543680, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 1756465152, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 1807386624, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1807400960, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1807403008, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 1808779264, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 1818413056, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1818427392, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.11.attn_v.bias", "offset": 1828061184, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1828063232, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 1829439488, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1829453824, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 1880375296, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 1931296768, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 1982218240, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1982232576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1982234624, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 1983610880, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 1993244672, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1993259008, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.12.attn_v.bias", "offset": 2002892800, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 2002894848, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 2004271104, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2004285440, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 2055206912, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 2106128384, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 2157049856, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 2157064192, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2157066240, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 2158442496, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 2168076288, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 2168090624, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.13.attn_v.bias", "offset": 2177724416, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 2177726464, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 2179102720, "shape": [ 3584, 18944 ], "size": 1376256, "type": 7 }, { "name": "blk.14.attn_k.bias", "offset": 2230024192, "shape": [ 512 ], "size": 50921472, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2230026240, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 2231402496, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 2241036288, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 2241050624, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.14.attn_v.bias", "offset": 2250684416, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 2250686464, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 2252062720, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2252077056, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 2302998528, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 2353920000, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 2404841472, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2404855808, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2404870144, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 2455791616, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 2506713088, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 2557634560, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2557648896, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2557650944, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 2559027200, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 2568660992, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2568675328, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.7.attn_v.bias", "offset": 2578309120, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2578311168, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 2579687424, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2579701760, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 2630623232, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 2681544704, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 2732466176, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2732480512, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2732482560, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 2733858816, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 2743492608, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 2743506944, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.8.attn_v.bias", "offset": 2753140736, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 2753142784, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 2754519040, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2754533376, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 2805454848, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 2856376320, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 2907297792, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 2907312128, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 2907314176, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 2908690432, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 2918324224, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 2918338560, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.9.attn_v.bias", "offset": 2927972352, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 2927974400, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 2929350656, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2929364992, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 2980286464, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 3031207936, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 3031222272, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3031236608, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 3082158080, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 3133079552, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 3184001024, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 3184015360, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3184017408, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 3185393664, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 3195027456, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 3195041792, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.15.attn_v.bias", "offset": 3204675584, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 3204677632, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 3206053888, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3206068224, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 3256989696, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 3307911168, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 3358832640, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 3358846976, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3358849024, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 3360225280, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 3369859072, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 3369873408, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.16.attn_v.bias", "offset": 3379507200, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 3379509248, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 3380885504, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3380899840, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 3431821312, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 3482742784, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 3533664256, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 3533678592, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3533680640, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 3535056896, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 3544690688, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 3544705024, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.17.attn_v.bias", "offset": 3554338816, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 3554340864, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 3555717120, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3555731456, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 3606652928, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 3657574400, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 3708495872, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 3708510208, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 3708512256, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 3709888512, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 3719522304, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 3719536640, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.18.attn_v.bias", "offset": 3729170432, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 3729172480, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 3730548736, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3730563072, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 3781484544, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 3832406016, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 3883327488, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 3883341824, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 3883343872, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 3884720128, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 3894353920, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 3894368256, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.19.attn_v.bias", "offset": 3904002048, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 3904004096, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 3905380352, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3905394688, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 3956316160, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 4007237632, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 4058159104, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 4058173440, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4058175488, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 4059551744, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 4069185536, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 4069199872, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.20.attn_v.bias", "offset": 4078833664, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 4078835712, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 4080211968, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4080226304, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 4131147776, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 4182069248, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 4232990720, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 4233005056, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4233007104, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 4234383360, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 4244017152, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 4244031488, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.21.attn_v.bias", "offset": 4253665280, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 4253667328, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 4255043584, "shape": [ 3584, 18944 ], "size": 1376256, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 4305965056, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.22.attn_k.bias", "offset": 4356886528, "shape": [ 512 ], "size": 50921472, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4356888576, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 4358264832, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 4367898624, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 4367912960, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.22.attn_v.bias", "offset": 4377546752, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 4377548800, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "output.weight", "offset": 4378925056, "shape": [ 3584, 152064 ], "size": 1376256, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 4825993216, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4826007552, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 4876929024, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 4876943360, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4876957696, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 4927879168, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 4978800640, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 5029722112, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 5029736448, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 5029738496, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 5031114752, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 5040748544, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 5040762880, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.23.attn_v.bias", "offset": 5050396672, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 5050398720, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 5051774976, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5051789312, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.24.ffn_gate.weight", "offset": 5102710784, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 5153632256, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 5204553728, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 5204568064, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 5204570112, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.24.attn_output.weight", "offset": 5205946368, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.24.attn_q.bias", "offset": 5215580160, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 5215594496, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.24.attn_v.bias", "offset": 5225228288, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 5225230336, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 5226606592, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5226620928, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.25.ffn_gate.weight", "offset": 5277542400, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 5328463872, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 5379385344, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 5379399680, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5379401728, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.25.attn_output.weight", "offset": 5380777984, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.25.attn_q.bias", "offset": 5390411776, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 5390426112, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.25.attn_v.bias", "offset": 5400059904, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 5400061952, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 5401438208, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5401452544, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.26.ffn_gate.weight", "offset": 5452374016, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 5503295488, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 5554216960, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 5554231296, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 5554233344, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.26.attn_output.weight", "offset": 5555609600, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.26.attn_q.bias", "offset": 5565243392, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 5565257728, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.26.attn_v.bias", "offset": 5574891520, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 5574893568, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 5576269824, "shape": [ 3584 ], "size": 1376256, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5576284160, "shape": [ 18944, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.27.ffn_gate.weight", "offset": 5627205632, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 5678127104, "shape": [ 3584, 18944 ], "size": 50921472, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 5729048576, "shape": [ 3584 ], "size": 50921472, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 5729062912, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 5729064960, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "blk.27.attn_output.weight", "offset": 5730441216, "shape": [ 3584, 3584 ], "size": 1376256, "type": 7 }, { "name": "blk.27.attn_q.bias", "offset": 5740075008, "shape": [ 3584 ], "size": 9633792, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 5740089344, "shape": [ 3584, 3584 ], "size": 14336, "type": 7 }, { "name": "blk.27.attn_v.bias", "offset": 5749723136, "shape": [ 512 ], "size": 9633792, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 5749725184, "shape": [ 3584, 512 ], "size": 2048, "type": 7 }, { "name": "output_norm.weight", "offset": 5751101440, "shape": [ 3584 ], "size": 1376256, "type": 0 } ], "version": 3 }