qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 10, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 178827264, "shape": [ 3584 ], "size": 178827264, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 178841600, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 208015360, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.0.ffn_up.weight", "offset": 230293504, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 252571648, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 252585984, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 252588032, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.0.attn_output.weight", "offset": 253190144, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.0.attn_q.bias", "offset": 258709504, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 258723840, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.0.attn_v.bias", "offset": 262938624, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 262940672, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 263972864, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 263987200, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 293160960, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.1.ffn_up.weight", "offset": 315439104, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 337717248, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 337731584, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 337733632, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.1.attn_output.weight", "offset": 338335744, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.1.attn_q.bias", "offset": 343855104, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 343869440, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.1.attn_v.bias", "offset": 348084224, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 348086272, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 349118464, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 349132800, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 378306560, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.2.ffn_up.weight", "offset": 400584704, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 422862848, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 422877184, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 422879232, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.2.attn_output.weight", "offset": 423481344, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.2.attn_q.bias", "offset": 429000704, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 429015040, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.2.attn_v.bias", "offset": 433229824, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 433231872, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 434264064, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 434278400, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 463452160, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.3.ffn_up.weight", "offset": 485730304, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 508008448, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 508022784, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 508024832, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.3.attn_output.weight", "offset": 508626944, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.3.attn_q.bias", "offset": 514146304, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 514160640, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.3.attn_v.bias", "offset": 518375424, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 518377472, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 519409664, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 519424000, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 548597760, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.4.ffn_up.weight", "offset": 570875904, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 593154048, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 593168384, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 593170432, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.4.attn_output.weight", "offset": 593772544, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.4.attn_q.bias", "offset": 599291904, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 599306240, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.4.attn_v.bias", "offset": 603521024, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000\u0000blk.16.attn_v.", "offset": 603523072, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 604555264, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 604569600, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 633743360, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.5.ffn_up.weight", "offset": 656021504, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 678299648, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 678313984, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 678316032, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.5.attn_output.weight", "offset": 678918144, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.5.attn_q.bias", "offset": 684437504, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 684451840, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.5.attn_v.bias", "offset": 688666624, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 688668672, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_k.bias", "offset": 689700864, "shape": [ 512 ], "size": 1032192, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 689702912, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 690305024, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.6.attn_q.bias", "offset": 695824384, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 695838720, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.6.attn_v.bias", "offset": 700053504, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 700055552, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 701087744, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 701102080, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 730275840, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.10.ffn_up.weight", "offset": 752553984, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 774832128, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 774846464, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 774848512, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.10.attn_output.weight", "offset": 775450624, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.10.attn_q.bias", "offset": 780969984, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 780984320, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.10.attn_v.bias", "offset": 785199104, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 785201152, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 786233344, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 786247680, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 815421440, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.11.ffn_up.weight", "offset": 837699584, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 859977728, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 859992064, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 859994112, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.11.attn_output.weight", "offset": 860596224, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.11.attn_q.bias", "offset": 866115584, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 866129920, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.11.attn_v.bias", "offset": 870344704, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 870346752, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 871378944, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 871393280, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 900567040, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.12.ffn_up.weight", "offset": 922845184, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 945123328, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 945137664, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 945139712, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.12.attn_output.weight", "offset": 945741824, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.12.attn_q.bias", "offset": 951261184, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 951275520, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.12.attn_v.bias", "offset": 955490304, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 955492352, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 956524544, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 956538880, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 985712640, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.13.ffn_up.weight", "offset": 1007990784, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 1030268928, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1030283264, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1030285312, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.13.attn_output.weight", "offset": 1030887424, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.13.attn_q.bias", "offset": 1036406784, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1036421120, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.13.attn_v.bias", "offset": 1040635904, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1040637952, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 1041670144, "shape": [ 3584, 18944 ], "size": 1032192, "type": 10 }, { "name": "blk.14.attn_k.bias", "offset": 1063948288, "shape": [ 512 ], "size": 22278144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1063950336, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.14.attn_output.weight", "offset": 1064552448, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.14.attn_q.bias", "offset": 1070071808, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1070086144, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.14.attn_v.bias", "offset": 1074300928, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1074302976, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 1075335168, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1075349504, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 1104523264, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.6.ffn_up.weight", "offset": 1126801408, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.6.ffn_norm.weight", "offset": 1149079552, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1149093888, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1149108224, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 1178281984, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.7.ffn_up.weight", "offset": 1200560128, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 1222838272, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1222852608, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1222854656, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.7.attn_output.weight", "offset": 1223456768, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.7.attn_q.bias", "offset": 1228976128, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1228990464, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.7.attn_v.bias", "offset": 1233205248, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1233207296, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 1234239488, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1234253824, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 1263427584, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.8.ffn_up.weight", "offset": 1285705728, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 1307983872, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1307998208, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1308000256, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.8.attn_output.weight", "offset": 1308602368, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.8.attn_q.bias", "offset": 1314121728, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1314136064, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.8.attn_v.bias", "offset": 1318350848, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1318352896, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 1319385088, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1319399424, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 1348573184, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.9.ffn_up.weight", "offset": 1370851328, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 1393129472, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1393143808, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1393145856, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.9.attn_output.weight", "offset": 1393747968, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.9.attn_q.bias", "offset": 1399267328, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1399281664, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.9.attn_v.bias", "offset": 1403496448, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1403498496, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 1404530688, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1404545024, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1433718784, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 1455996928, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 1456011264, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1456025600, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 1485199360, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.15.ffn_up.weight", "offset": 1507477504, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 1529755648, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 1529769984, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1529772032, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.15.attn_output.weight", "offset": 1530374144, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.15.attn_q.bias", "offset": 1535893504, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 1535907840, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.15.attn_v.bias", "offset": 1540122624, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 1540124672, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 1541156864, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1541171200, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 1570344960, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.16.ffn_up.weight", "offset": 1592623104, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 1614901248, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 1614915584, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1614917632, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.16.attn_output.weight", "offset": 1615519744, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.16.attn_q.bias", "offset": 1621039104, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1621053440, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.16.attn_v.bias", "offset": 1625268224, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "\f\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdi}iY\ufffde\ufffd\ufffd^\u003eZV*", "offset": 1625270272, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 1626302464, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1626316800, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 1655490560, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.17.ffn_up.weight", "offset": 1677768704, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 1700046848, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 1700061184, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 1700063232, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.17.attn_output.weight", "offset": 1700665344, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.17.attn_q.bias", "offset": 1706184704, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 1706199040, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.17.attn_v.bias", "offset": 1710413824, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 1710415872, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 1711448064, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 1711462400, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 1740636160, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.18.ffn_up.weight", "offset": 1762914304, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 1785192448, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 1785206784, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 1785208832, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.18.attn_output.weight", "offset": 1785810944, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.18.attn_q.bias", "offset": 1791330304, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 1791344640, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.18.attn_v.bias", "offset": 1795559424, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 1795561472, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 1796593664, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 1796608000, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 1825781760, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.19.ffn_up.weight", "offset": 1848059904, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 1870338048, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 1870352384, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 1870354432, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.19.attn_output.weight", "offset": 1870956544, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.19.attn_q.bias", "offset": 1876475904, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 1876490240, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.19.attn_v.bias", "offset": 1880705024, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 1880707072, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 1881739264, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 1881753600, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 1910927360, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.20.ffn_up.weight", "offset": 1933205504, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 1955483648, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 1955497984, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 1955500032, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.20.attn_output.weight", "offset": 1956102144, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.20.attn_q.bias", "offset": 1961621504, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 1961635840, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.20.attn_v.bias", "offset": 1965850624, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 1965852672, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 1966884864, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1966899200, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 1996072960, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.21.ffn_up.weight", "offset": 2018351104, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 2040629248, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 2040643584, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2040645632, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 2041247744, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.21.attn_q.bias", "offset": 2046767104, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2046781440, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.21.attn_v.bias", "offset": 2050996224, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 2050998272, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 2052030464, "shape": [ 3584, 18944 ], "size": 1032192, "type": 10 }, { "name": "blk.22.ffn_up.weight", "offset": 2074308608, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.22.attn_k.bias", "offset": 2096586752, "shape": [ 512 ], "size": 22278144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2096588800, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.22.attn_output.weight", "offset": 2097190912, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.22.attn_q.bias", "offset": 2102710272, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2102724608, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.22.attn_v.bias", "offset": 2106939392, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 2106941440, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "output.weight", "offset": 2107973632, "shape": [ 3584, 152064 ], "size": 1032192, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 2555041792, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2555056128, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 2584229888, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 2584244224, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2584258560, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 2613432320, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.23.ffn_up.weight", "offset": 2635710464, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 2657988608, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 2658002944, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 2658004992, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.23.attn_output.weight", "offset": 2658607104, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.23.attn_q.bias", "offset": 2664126464, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 2664140800, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.23.attn_v.bias", "offset": 2668355584, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 2668357632, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 2669389824, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 2669404160, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 2698577920, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.24.ffn_up.weight", "offset": 2720856064, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 2743134208, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 2743148544, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 2743150592, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.24.attn_output.weight", "offset": 2743752704, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.24.attn_q.bias", "offset": 2749272064, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 2749286400, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.24.attn_v.bias", "offset": 2753501184, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 2753503232, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 2754535424, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 2754549760, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 2783723520, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.25.ffn_up.weight", "offset": 2806001664, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 2828279808, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 2828294144, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 2828296192, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.25.attn_output.weight", "offset": 2828898304, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.25.attn_q.bias", "offset": 2834417664, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 2834432000, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.25.attn_v.bias", "offset": 2838646784, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 2838648832, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 2839681024, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 2839695360, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 2868869120, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.26.ffn_up.weight", "offset": 2891147264, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 2913425408, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 2913439744, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 2913441792, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.26.attn_output.weight", "offset": 2914043904, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.26.attn_q.bias", "offset": 2919563264, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 2919577600, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.26.attn_v.bias", "offset": 2923792384, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 2923794432, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 2924826624, "shape": [ 3584 ], "size": 1032192, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2924840960, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 2954014720, "shape": [ 3584, 18944 ], "size": 29173760, "type": 10 }, { "name": "blk.27.ffn_up.weight", "offset": 2976292864, "shape": [ 3584, 18944 ], "size": 22278144, "type": 10 }, { "name": "blk.27.ffn_norm.weight", "offset": 2998571008, "shape": [ 3584 ], "size": 22278144, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 2998585344, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 2998587392, "shape": [ 3584, 512 ], "size": 2048, "type": 10 }, { "name": "blk.27.attn_output.weight", "offset": 2999189504, "shape": [ 3584, 3584 ], "size": 602112, "type": 11 }, { "name": "blk.27.attn_q.bias", "offset": 3004708864, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 3004723200, "shape": [ 3584, 3584 ], "size": 14336, "type": 10 }, { "name": "blk.27.attn_v.bias", "offset": 3008937984, "shape": [ 512 ], "size": 4214784, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 3008940032, "shape": [ 3584, 512 ], "size": 2048, "type": 12 }, { "name": "output_norm.weight", "offset": 3009972224, "shape": [ 3584 ], "size": 1032192, "type": 0 } ], "version": 3 }