qwen2

Qwen2 is a new series of large language models from Alibaba group

0.5B 1.5B 7B 72B

273.4K Pulls Updated 4 weeks ago

97 Tags

... /
model
82e3445ca8bb · 8.1GB
{ "metadata": { "general.architecture": "qwen2", "general.file_type": 7, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 579059712, "shape": [ 3584 ], "size": 579059712, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 579074048, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 651212800, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 723351552, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 795490304, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 795504640, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 795506688, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 797456384, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.0.attn_q.bias", "offset": 811104256, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 811118592, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.0.attn_v.bias", "offset": 824766464, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 824768512, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 826718208, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 826732544, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 898871296, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 971010048, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 1043148800, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 1043163136, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 1043165184, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 1045114880, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.1.attn_q.bias", "offset": 1058762752, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 1058777088, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.1.attn_v.bias", "offset": 1072424960, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 1072427008, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 1074376704, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 1074391040, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 1146529792, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 1218668544, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 1290807296, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 1290821632, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 1290823680, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 1292773376, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.2.attn_q.bias", "offset": 1306421248, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1306435584, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.2.attn_v.bias", "offset": 1320083456, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 1320085504, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 1322035200, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1322049536, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 1394188288, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 1466327040, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 1538465792, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1538480128, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1538482176, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 1540431872, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.3.attn_q.bias", "offset": 1554079744, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1554094080, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.3.attn_v.bias", "offset": 1567741952, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1567744000, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 1569693696, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1569708032, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 1641846784, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 1713985536, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 1786124288, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1786138624, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1786140672, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 1788090368, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.4.attn_q.bias", "offset": 1801738240, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1801752576, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.4.attn_v.bias", "offset": 1815400448, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000\u0000blk.16.attn_v.", "offset": 1815402496, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 1817352192, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1817366528, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 1889505280, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 1961644032, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 2033782784, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 2033797120, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 2033799168, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 2035748864, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.5.attn_q.bias", "offset": 2049396736, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 2049411072, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.5.attn_v.bias", "offset": 2063058944, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 2063060992, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.6.attn_k.bias", "offset": 2065010688, "shape": [ 512 ], "size": 1949696, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 2065012736, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 2066962432, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.6.attn_q.bias", "offset": 2080610304, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 2080624640, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.6.attn_v.bias", "offset": 2094272512, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 2094274560, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 2096224256, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2096238592, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 2168377344, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 2240516096, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 2312654848, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 2312669184, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 2312671232, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 2314620928, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.10.attn_q.bias", "offset": 2328268800, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 2328283136, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.10.attn_v.bias", "offset": 2341931008, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 2341933056, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 2343882752, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2343897088, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 2416035840, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 2488174592, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 2560313344, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 2560327680, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 2560329728, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 2562279424, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.11.attn_q.bias", "offset": 2575927296, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 2575941632, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.11.attn_v.bias", "offset": 2589589504, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 2589591552, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 2591541248, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2591555584, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 2663694336, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 2735833088, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 2807971840, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 2807986176, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 2807988224, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 2809937920, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.12.attn_q.bias", "offset": 2823585792, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 2823600128, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.12.attn_v.bias", "offset": 2837248000, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 2837250048, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 2839199744, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2839214080, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 2911352832, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 2983491584, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 3055630336, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 3055644672, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 3055646720, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 3057596416, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.13.attn_q.bias", "offset": 3071244288, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 3071258624, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.13.attn_v.bias", "offset": 3084906496, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 3084908544, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 3086858240, "shape": [ 3584, 18944 ], "size": 1949696, "type": 8 }, { "name": "blk.14.attn_k.bias", "offset": 3158996992, "shape": [ 512 ], "size": 72138752, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 3158999040, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 3160948736, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.14.attn_q.bias", "offset": 3174596608, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 3174610944, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.14.attn_v.bias", "offset": 3188258816, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 3188260864, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 3190210560, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3190224896, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 3262363648, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 3334502400, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 3406641152, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 3406655488, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 3406669824, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 3478808576, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 3550947328, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 3623086080, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 3623100416, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 3623102464, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 3625052160, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.7.attn_q.bias", "offset": 3638700032, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 3638714368, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.7.attn_v.bias", "offset": 3652362240, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 3652364288, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 3654313984, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3654328320, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 3726467072, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 3798605824, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 3870744576, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 3870758912, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 3870760960, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 3872710656, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.8.attn_q.bias", "offset": 3886358528, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 3886372864, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.8.attn_v.bias", "offset": 3900020736, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 3900022784, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 3901972480, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3901986816, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 3974125568, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 4046264320, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 4118403072, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 4118417408, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 4118419456, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 4120369152, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.9.attn_q.bias", "offset": 4134017024, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 4134031360, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.9.attn_v.bias", "offset": 4147679232, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 4147681280, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 4149630976, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4149645312, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 4221784064, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 4293922816, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 4293937152, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 4293951488, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 4366090240, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 4438228992, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 4510367744, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 4510382080, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 4510384128, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 4512333824, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.15.attn_q.bias", "offset": 4525981696, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 4525996032, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.15.attn_v.bias", "offset": 4539643904, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 4539645952, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 4541595648, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4541609984, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 4613748736, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 4685887488, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 4758026240, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 4758040576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 4758042624, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 4759992320, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.16.attn_q.bias", "offset": 4773640192, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 4773654528, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.16.attn_v.bias", "offset": 4787302400, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "2\ufffd\ufffd#*\ufffd\ufffd,\n}\ufffd\u0001\ufffdg\u0002+\u0011ED\u0006", "offset": 4787304448, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 4789254144, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4789268480, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 4861407232, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 4933545984, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 5005684736, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 5005699072, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 5005701120, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 5007650816, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.17.attn_q.bias", "offset": 5021298688, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 5021313024, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.17.attn_v.bias", "offset": 5034960896, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 5034962944, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 5036912640, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 5036926976, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 5109065728, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 5181204480, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 5253343232, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 5253357568, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 5253359616, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 5255309312, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.18.attn_q.bias", "offset": 5268957184, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 5268971520, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.18.attn_v.bias", "offset": 5282619392, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 5282621440, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 5284571136, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 5284585472, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 5356724224, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 5428862976, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 5501001728, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 5501016064, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 5501018112, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 5502967808, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.19.attn_q.bias", "offset": 5516615680, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 5516630016, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.19.attn_v.bias", "offset": 5530277888, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 5530279936, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 5532229632, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5532243968, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 5604382720, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 5676521472, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 5748660224, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 5748674560, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 5748676608, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 5750626304, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.20.attn_q.bias", "offset": 5764274176, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 5764288512, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.20.attn_v.bias", "offset": 5777936384, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 5777938432, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 5779888128, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5779902464, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 5852041216, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 5924179968, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 5996318720, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 5996333056, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 5996335104, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 5998284800, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.21.attn_q.bias", "offset": 6011932672, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 6011947008, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.21.attn_v.bias", "offset": 6025594880, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 6025596928, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 6027546624, "shape": [ 3584, 18944 ], "size": 1949696, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 6099685376, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.22.attn_k.bias", "offset": 6171824128, "shape": [ 512 ], "size": 72138752, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 6171826176, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 6173775872, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.22.attn_q.bias", "offset": 6187423744, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 6187438080, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.22.attn_v.bias", "offset": 6201085952, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 6201088000, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "output.weight", "offset": 6203037696, "shape": [ 3584, 152064 ], "size": 1949696, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 6782097408, "shape": [ 3584 ], "size": 579059712, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 6782111744, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 6854250496, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 6854264832, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6854279168, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 6926417920, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 6998556672, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 7070695424, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 7070709760, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 7070711808, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 7072661504, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.23.attn_q.bias", "offset": 7086309376, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 7086323712, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.23.attn_v.bias", "offset": 7099971584, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 7099973632, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 7101923328, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 7101937664, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 7174076416, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 7246215168, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 7318353920, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 7318368256, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 7318370304, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 7320320000, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.24.attn_q.bias", "offset": 7333967872, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 7333982208, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.24.attn_v.bias", "offset": 7347630080, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 7347632128, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 7349581824, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 7349596160, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 7421734912, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 7493873664, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 7566012416, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 7566026752, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 7566028800, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 7567978496, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.25.attn_q.bias", "offset": 7581626368, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 7581640704, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.25.attn_v.bias", "offset": 7595288576, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 7595290624, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 7597240320, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 7597254656, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 7669393408, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 7741532160, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 7813670912, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 7813685248, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 7813687296, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 7815636992, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.26.attn_q.bias", "offset": 7829284864, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 7829299200, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.26.attn_v.bias", "offset": 7842947072, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 7842949120, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 7844898816, "shape": [ 3584 ], "size": 1949696, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 7844913152, "shape": [ 18944, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 7917051904, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 7989190656, "shape": [ 3584, 18944 ], "size": 72138752, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 8061329408, "shape": [ 3584 ], "size": 72138752, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 8061343744, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 8061345792, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 8063295488, "shape": [ 3584, 3584 ], "size": 1949696, "type": 8 }, { "name": "blk.27.attn_q.bias", "offset": 8076943360, "shape": [ 3584 ], "size": 13647872, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 8076957696, "shape": [ 3584, 3584 ], "size": 14336, "type": 8 }, { "name": "blk.27.attn_v.bias", "offset": 8090605568, "shape": [ 512 ], "size": 13647872, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 8090607616, "shape": [ 3584, 512 ], "size": 2048, "type": 8 }, { "name": "output_norm.weight", "offset": 8092557312, "shape": [ 3584 ], "size": 1949696, "type": 0 } ], "version": 3 }