qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 18, "general.name": "Qwen2-7B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 28, "qwen2.attention.head_count_kv": 4, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 3584, "qwen2.feed_forward_length": 18944, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (152064 values)", "tokenizer.ggml.tokens": "... (152064 values)" }, "num_params": 7615616512, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 447068160, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 447082496, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 502777856, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 558473216, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.0.ffn_norm.weight", "offset": 614168576, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 614182912, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 614184960, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 615690240, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.0.attn_q.bias", "offset": 626227200, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 626241536, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.0.attn_v.bias", "offset": 636778496, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 636780544, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 638285824, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 638300160, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 693995520, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 749690880, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 805386240, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 805400576, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 805402624, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 806907904, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.1.attn_q.bias", "offset": 817444864, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 817459200, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.1.attn_v.bias", "offset": 827996160, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 827998208, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 829503488, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 829517824, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 885213184, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 940908544, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 996603904, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 996618240, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 996620288, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 998125568, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.2.attn_q.bias", "offset": 1008662528, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 1008676864, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.2.attn_v.bias", "offset": 1019213824, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 1019215872, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 1020721152, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1020735488, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1076430848, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 1132126208, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 1187821568, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1187835904, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1187837952, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 1189343232, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.3.attn_q.bias", "offset": 1199880192, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1199894528, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.3.attn_v.bias", "offset": 1210431488, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1210433536, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1211938816, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1211953152, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1267648512, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1323343872, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 1379039232, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1379053568, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1379055616, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 1380560896, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.4.attn_q.bias", "offset": 1391097856, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1391112192, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.4.attn_v.bias", "offset": 1401649152, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1401651200, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1403156480, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1403170816, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 1458866176, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 1514561536, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 1570256896, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1570271232, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1570273280, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 1571778560, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.5.attn_q.bias", "offset": 1582315520, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1582329856, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.5.attn_v.bias", "offset": 1592866816, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1592868864, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_k.bias", "offset": 1594374144, "shape": [ 512 ], "size": 1505280, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1594376192, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 1595881472, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.6.attn_q.bias", "offset": 1606418432, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1606432768, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.6.attn_v.bias", "offset": 1616969728, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1616971776, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 1618477056, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1618491392, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 1674186752, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 1729882112, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 1785577472, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 1785591808, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 1785593856, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 1787099136, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.10.attn_q.bias", "offset": 1797636096, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 1797650432, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.10.attn_v.bias", "offset": 1808187392, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 1808189440, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 1809694720, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1809709056, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 1865404416, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 1921099776, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 1976795136, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 1976809472, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 1976811520, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 1978316800, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.11.attn_q.bias", "offset": 1988853760, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 1988868096, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.11.attn_v.bias", "offset": 1999405056, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 1999407104, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 2000912384, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2000926720, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 2056622080, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 2112317440, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 2168012800, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 2168027136, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 2168029184, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 2169534464, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.12.attn_q.bias", "offset": 2180071424, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 2180085760, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.12.attn_v.bias", "offset": 2190622720, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 2190624768, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 2192130048, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2192144384, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 2247839744, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 2303535104, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 2359230464, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 2359244800, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 2359246848, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 2360752128, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.13.attn_q.bias", "offset": 2371289088, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 2371303424, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.13.attn_v.bias", "offset": 2381840384, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 2381842432, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 2383347712, "shape": [ 3584, 18944 ], "size": 1505280, "type": 14 }, { "name": "blk.14.attn_k.bias", "offset": 2439043072, "shape": [ 512 ], "size": 55695360, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 2439045120, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 2440550400, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.14.attn_q.bias", "offset": 2451087360, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 2451101696, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.14.attn_v.bias", "offset": 2461638656, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 2461640704, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 2463145984, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2463160320, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 2518855680, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 2574551040, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 2630246400, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2630260736, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2630275072, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 2685970432, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 2741665792, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 2797361152, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 2797375488, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 2797377536, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 2798882816, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.7.attn_q.bias", "offset": 2809419776, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 2809434112, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.7.attn_v.bias", "offset": 2819971072, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 2819973120, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 2821478400, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2821492736, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 2877188096, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 2932883456, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 2988578816, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 2988593152, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 2988595200, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 2990100480, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.8.attn_q.bias", "offset": 3000637440, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 3000651776, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.8.attn_v.bias", "offset": 3011188736, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 3011190784, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 3012696064, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3012710400, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 3068405760, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 3124101120, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.9.ffn_norm.weight", "offset": 3179796480, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 3179810816, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 3179812864, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 3181318144, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.9.attn_q.bias", "offset": 3191855104, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 3191869440, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.9.attn_v.bias", "offset": 3202406400, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 3202408448, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 3203913728, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3203928064, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 3259623424, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 3315318784, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.15.attn_norm.weight", "offset": 3315333120, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3315347456, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 3371042816, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 3426738176, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 3482433536, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 3482447872, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 3482449920, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 3483955200, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.15.attn_q.bias", "offset": 3494492160, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 3494506496, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.15.attn_v.bias", "offset": 3505043456, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 3505045504, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 3506550784, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3506565120, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 3562260480, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 3617955840, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 3673651200, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 3673665536, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 3673667584, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 3675172864, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.16.attn_q.bias", "offset": 3685709824, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 3685724160, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "\ufffdi\ufffd\ufffdTEjHbfx\ufffd\ufffd\ufffd9ci\ufffd", "offset": 3696261120, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 3696263168, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 3697768448, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3697782784, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 3753478144, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 3809173504, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 3864868864, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 3864883200, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 3864885248, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 3866390528, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.17.attn_q.bias", "offset": 3876927488, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 3876941824, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.17.attn_v.bias", "offset": 3887478784, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 3887480832, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 3888986112, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3889000448, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 3944695808, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 4000391168, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 4056086528, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 4056100864, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 4056102912, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 4057608192, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.18.attn_q.bias", "offset": 4068145152, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 4068159488, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.18.attn_v.bias", "offset": 4078696448, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 4078698496, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 4080203776, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4080218112, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 4135913472, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 4191608832, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 4247304192, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 4247318528, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 4247320576, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 4248825856, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.19.attn_q.bias", "offset": 4259362816, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 4259377152, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.19.attn_v.bias", "offset": 4269914112, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 4269916160, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 4271421440, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4271435776, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 4327131136, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 4382826496, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 4438521856, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 4438536192, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 4438538240, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 4440043520, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.20.attn_q.bias", "offset": 4450580480, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 4450594816, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.20.attn_v.bias", "offset": 4461131776, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 4461133824, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 4462639104, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4462653440, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 4518348800, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 4574044160, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 4629739520, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 4629753856, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 4629755904, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 4631261184, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.21.attn_q.bias", "offset": 4641798144, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 4641812480, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.21.attn_v.bias", "offset": 4652349440, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 4652351488, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 4653856768, "shape": [ 3584, 18944 ], "size": 1505280, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 4709552128, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.22.attn_k.bias", "offset": 4765247488, "shape": [ 512 ], "size": 55695360, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 4765249536, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 4766754816, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.22.attn_q.bias", "offset": 4777291776, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 4777306112, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.22.attn_v.bias", "offset": 4787843072, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 4787845120, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "output.weight", "offset": 4789350400, "shape": [ 3584, 152064 ], "size": 1505280, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 5236418560, "shape": [ 3584 ], "size": 447068160, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5236432896, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 5292128256, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.23.attn_norm.weight", "offset": 5292142592, "shape": [ 3584 ], "size": 14336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5292156928, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 5347852288, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 5403547648, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 5459243008, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 5459257344, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 5459259392, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 5460764672, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.23.attn_q.bias", "offset": 5471301632, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 5471315968, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.23.attn_v.bias", "offset": 5481852928, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 5481854976, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 5483360256, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5483374592, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 5539069952, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 5594765312, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 5650460672, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 5650475008, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 5650477056, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 5651982336, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.24.attn_q.bias", "offset": 5662519296, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 5662533632, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.24.attn_v.bias", "offset": 5673070592, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 5673072640, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 5674577920, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5674592256, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 5730287616, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 5785982976, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 5841678336, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 5841692672, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 5841694720, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 5843200000, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.25.attn_q.bias", "offset": 5853736960, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 5853751296, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.25.attn_v.bias", "offset": 5864288256, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 5864290304, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 5865795584, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5865809920, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 5921505280, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 5977200640, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 6032896000, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 6032910336, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 6032912384, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 6034417664, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.26.attn_q.bias", "offset": 6044954624, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 6044968960, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.26.attn_v.bias", "offset": 6055505920, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 6055507968, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 6057013248, "shape": [ 3584 ], "size": 1505280, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6057027584, "shape": [ 18944, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 6112722944, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 6168418304, "shape": [ 3584, 18944 ], "size": 55695360, "type": 14 }, { "name": "blk.27.ffn_norm.weight", "offset": 6224113664, "shape": [ 3584 ], "size": 55695360, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 6224128000, "shape": [ 512 ], "size": 14336, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 6224130048, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 6225635328, "shape": [ 3584, 3584 ], "size": 1505280, "type": 14 }, { "name": "blk.27.attn_q.bias", "offset": 6236172288, "shape": [ 3584 ], "size": 10536960, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 6236186624, "shape": [ 3584, 3584 ], "size": 14336, "type": 14 }, { "name": "blk.27.attn_v.bias", "offset": 6246723584, "shape": [ 512 ], "size": 10536960, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 6246725632, "shape": [ 3584, 512 ], "size": 2048, "type": 14 }, { "name": "output_norm.weight", "offset": 6248230912, "shape": [ 3584 ], "size": 1505280, "type": 0 } ], "version": 3 }