qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 11, "general.name": "Qwen2-7B", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 131072, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151643, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 234178560, "shape": [ 3584 ], "size": 234178560, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 234192896, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 263366656, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 292540416, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 321714176, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 321728512, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 321730560, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 322519040, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.0.attn_q.bias", "offset": 328038400, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 328052736, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 333572096, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 333574144, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 334362624, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 334376960, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 363550720, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 392724480, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 421898240, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 421912576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 421914624, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 422703104, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.1.attn_q.bias", "offset": 428222464, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 428236800, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 433756160, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 433758208, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 434546688, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 434561024, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 463734784, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 492908544, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 522082304, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 522096640, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 522098688, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 522887168, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.2.attn_q.bias", "offset": 528406528, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 528420864, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 533940224, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 533942272, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 534730752, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 534745088, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 563918848, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 593092608, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 622266368, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 622280704, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 622282752, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 623071232, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.3.attn_q.bias", "offset": 628590592, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 628604928, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 634124288, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 634126336, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 634914816, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 634929152, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 664102912, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 693276672, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 722450432, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 722464768, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 722466816, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 723255296, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.4.attn_q.bias", "offset": 728774656, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 728788992, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 734308352, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 734310400, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 735098880, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 735113216, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 764286976, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 793460736, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 822634496, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 822648832, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 822650880, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 823439360, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.5.attn_q.bias", "offset": 828958720, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 828973056, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 834492416, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 834494464, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_k.bias", "offset": 835282944, "shape": [ 512 ], "size": 788480, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 835284992, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 836073472, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.6.attn_q.bias", "offset": 841592832, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 841607168, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 847126528, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 847128576, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 847917056, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 847931392, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 877105152, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 906278912, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 935452672, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 935467008, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 935469056, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 936257536, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.10.attn_q.bias", "offset": 941776896, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 941791232, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 947310592, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 947312640, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 948101120, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 948115456, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 977289216, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1006462976, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1035636736, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1035651072, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1035653120, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 1036441600, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.11.attn_q.bias", "offset": 1041960960, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1041975296, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 1047494656, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1047496704, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 1048285184, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1048299520, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 1077473280, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1106647040, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1135820800, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 1135835136, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 1135837184, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 1136625664, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.12.attn_q.bias", "offset": 1142145024, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 1142159360, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 1147678720, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 1147680768, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 1148469248, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1148483584, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 1177657344, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1206831104, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1236004864, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 1236019200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 1236021248, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 1236809728, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.13.attn_q.bias", "offset": 1242329088, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 1242343424, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 1247862784, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 1247864832, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 1248653312, "shape": [ 3584, 18944 ], "size": 788480, "type": 11 }, { "name": "blk.14.attn_k.bias", "offset": 1277827072, "shape": [ 512 ], "size": 29173760, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 1277829120, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 1278617600, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0000blk.22.ffn_d", "offset": 1284136960, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 1284151296, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 1289670656, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 1289672704, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1290461184, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1290475520, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 1319649280, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 1348823040, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1377996800, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1378011136, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1378025472, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 1407199232, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1436372992, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1465546752, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1465561088, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1465563136, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 1466351616, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.7.attn_q.bias", "offset": 1471870976, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1471885312, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 1477404672, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1477406720, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 1478195200, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1478209536, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 1507383296, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1536557056, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1565730816, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1565745152, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1565747200, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 1566535680, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.8.attn_q.bias", "offset": 1572055040, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1572069376, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 1577588736, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1577590784, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 1578379264, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1578393600, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 1607567360, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1636741120, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 1665914880, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1665929216, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1665931264, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 1666719744, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.9.attn_q.bias", "offset": 1672239104, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1672253440, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 1677772800, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1677774848, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 1678563328, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1678577664, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1707751424, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 1736925184, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 1736939520, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1736953856, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 1766127616, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1795301376, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 1824475136, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 1824489472, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 1824491520, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 1825280000, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.15.attn_q.bias", "offset": 1830799360, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 1830813696, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 1836333056, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 1836335104, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 1837123584, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 1837137920, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 1866311680, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 1895485440, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 1924659200, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 1924673536, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 1924675584, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 1925464064, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.16.attn_q.bias", "offset": 1930983424, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 1930997760, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 1936517120, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 1936519168, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 1937307648, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 1937321984, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 1966495744, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 1995669504, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 2024843264, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 2024857600, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 2024859648, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 2025648128, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.17.attn_q.bias", "offset": 2031167488, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 2031181824, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 2036701184, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 2036703232, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 2037491712, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2037506048, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 2066679808, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2095853568, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 2125027328, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 2125041664, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 2125043712, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 2125832192, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.18.attn_q.bias", "offset": 2131351552, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 2131365888, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 2136885248, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 2136887296, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 2137675776, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2137690112, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 2166863872, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2196037632, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 2225211392, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 2225225728, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 2225227776, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 2226016256, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.19.attn_q.bias", "offset": 2231535616, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 2231549952, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 2237069312, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 2237071360, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 2237859840, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2237874176, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 2267047936, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2296221696, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 2325395456, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 2325409792, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 2325411840, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 2326200320, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.20.attn_q.bias", "offset": 2331719680, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 2331734016, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 2337253376, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 2337255424, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 2338043904, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2338058240, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 2367232000, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2396405760, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 2425579520, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 2425593856, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 2425595904, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 2426384384, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.21.attn_q.bias", "offset": 2431903744, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 2431918080, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 2437437440, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 2437439488, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 2438227968, "shape": [ 3584, 18944 ], "size": 788480, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2467401728, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.22.attn_k.bias", "offset": 2496575488, "shape": [ 512 ], "size": 29173760, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 2496577536, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 2497366016, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.22.attn_q.bias", "offset": 2502885376, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 2502899712, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 2508419072, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 2508421120, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "output.weight", "offset": 2509209600, "shape": [ 3584, 152064 ], "size": 788480, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 2956277760, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2956292096, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 2985465856, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 2985480192, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2985494528, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 3014668288, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 3043842048, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 3073015808, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 3073030144, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 3073032192, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 3073820672, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.23.attn_q.bias", "offset": 3079340032, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 3079354368, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 3084873728, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 3084875776, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 3085664256, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3085678592, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 3114852352, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3144026112, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 3173199872, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 3173214208, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 3173216256, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 3174004736, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.24.attn_q.bias", "offset": 3179524096, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 3179538432, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 3185057792, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 3185059840, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 3185848320, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3185862656, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 3215036416, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3244210176, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 3273383936, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 3273398272, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 3273400320, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 3274188800, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.25.attn_q.bias", "offset": 3279708160, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 3279722496, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 3285241856, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 3285243904, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 3286032384, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3286046720, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 3315220480, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3344394240, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 3373568000, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 3373582336, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 3373584384, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 3374372864, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.26.attn_q.bias", "offset": 3379892224, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 3379906560, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 3385425920, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 3385427968, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 3386216448, "shape": [ 3584 ], "size": 788480, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3386230784, "shape": [ 18944, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 3415404544, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 3444578304, "shape": [ 3584, 18944 ], "size": 29173760, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 3473752064, "shape": [ 3584 ], "size": 29173760, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 3473766400, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 3473768448, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 3474556928, "shape": [ 3584, 3584 ], "size": 788480, "type": 11 }, { "name": "blk.27.attn_q.bias", "offset": 3480076288, "shape": [ 3584 ], "size": 5519360, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 3480090624, "shape": [ 3584, 3584 ], "size": 14336, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 3485609984, "shape": [ 512 ], "size": 5519360, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 3485612032, "shape": [ 3584, 512 ], "size": 2048, "type": 11 }, { "name": "output_norm.weight", "offset": 3486400512, "shape": [ 3584 ], "size": 788480, "type": 0 } ], "version": 3 }