qwen2

Qwen2 is a new series of large language models from Alibaba group

0.5B 1.5B 7B 72B

273.3K Pulls Updated 4 weeks ago

97 Tags

... /
model
b4b8cc447dcb · 1.6GB
{ "metadata": { "general.architecture": "qwen2", "general.file_type": 7, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 247959552, "shape": [ 1536 ], "size": 247959552, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 247965696, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.0.ffn_gate.weight", "offset": 262588416, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 277211136, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 291833856, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 291840000, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 291841024, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 292258816, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.0.attn_q.bias", "offset": 294765568, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 294771712, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.0.attn_v.bias", "offset": 297278464, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 297279488, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 297697280, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 297703424, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.1.ffn_gate.weight", "offset": 312326144, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 326948864, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 341571584, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 341577728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 341578752, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 341996544, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.1.attn_q.bias", "offset": 344503296, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 344509440, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.1.attn_v.bias", "offset": 347016192, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 347017216, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 347435008, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 347441152, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.10.ffn_gate.weight", "offset": 362063872, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 376686592, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 391309312, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 391315456, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 391316480, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 391734272, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.10.attn_q.bias", "offset": 394241024, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 394247168, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.10.attn_v.bias", "offset": 396753920, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 396754944, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 397172736, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 397178880, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.11.ffn_gate.weight", "offset": 411801600, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 426424320, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 441047040, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 441053184, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 441054208, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 441472000, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.11.attn_q.bias", "offset": 443978752, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 443984896, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.11.attn_v.bias", "offset": 446491648, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 446492672, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 446910464, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 446916608, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.12.ffn_gate.weight", "offset": 461539328, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 476162048, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 490784768, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 490790912, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 490791936, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 491209728, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.12.attn_q.bias", "offset": 493716480, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 493722624, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.12.attn_v.bias", "offset": 496229376, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 496230400, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 496648192, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 496654336, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.13.ffn_gate.weight", "offset": 511277056, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 525899776, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 540522496, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 540528640, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 540529664, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 540947456, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.13.attn_q.bias", "offset": 543454208, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 543460352, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.13.attn_v.bias", "offset": 545967104, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 545968128, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 546385920, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 546392064, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.14.ffn_gate.weight", "offset": 561014784, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 575637504, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 590260224, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 590266368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 590267392, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 590685184, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.14.attn_q.bias", "offset": 593191936, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 593198080, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.14.attn_v.bias", "offset": 595704832, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 595705856, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 596123648, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 596129792, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.15.ffn_gate.weight", "offset": 610752512, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 625375232, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 639997952, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 640004096, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 640005120, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 640422912, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.15.attn_q.bias", "offset": 642929664, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 642935808, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.15.attn_v.bias", "offset": 645442560, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 645443584, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 645861376, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 645867520, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.16.ffn_gate.weight", "offset": 660490240, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 675112960, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 689735680, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 689741824, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 689742848, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 690160640, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.16.attn_q.bias", "offset": 692667392, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 692673536, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.16.attn_v.bias", "offset": 695180288, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 695181312, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 695599104, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 695605248, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.17.ffn_gate.weight", "offset": 710227968, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 724850688, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 739473408, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 739479552, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 739480576, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 739898368, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.17.attn_q.bias", "offset": 742405120, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 742411264, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.17.attn_v.bias", "offset": 744918016, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 744919040, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 745336832, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 745342976, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.18.ffn_gate.weight", "offset": 759965696, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 774588416, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 789211136, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 789217280, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 789218304, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 789636096, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.18.attn_q.bias", "offset": 792142848, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 792148992, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.18.attn_v.bias", "offset": 794655744, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 794656768, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 795074560, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 795080704, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.19.ffn_gate.weight", "offset": 809703424, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 824326144, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 838948864, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 838955008, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 838956032, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 839373824, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.19.attn_q.bias", "offset": 841880576, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 841886720, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.19.attn_v.bias", "offset": 844393472, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 844394496, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 844812288, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 844818432, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.2.ffn_gate.weight", "offset": 859441152, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 874063872, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 888686592, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 888692736, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 888693760, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 889111552, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.2.attn_q.bias", "offset": 891618304, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 891624448, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.2.attn_v.bias", "offset": 894131200, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 894132224, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 894550016, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 894556160, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.20.ffn_gate.weight", "offset": 909178880, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 923801600, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 938424320, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 938430464, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 938431488, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 938849280, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.20.attn_q.bias", "offset": 941356032, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 941362176, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.20.attn_v.bias", "offset": 943868928, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 943869952, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 944287744, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 944293888, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.21.ffn_gate.weight", "offset": 958916608, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 973539328, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 988162048, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 988168192, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 988169216, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 988587008, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.21.attn_q.bias", "offset": 991093760, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 991099904, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.21.attn_v.bias", "offset": 993606656, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 993607680, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 994025472, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 994031616, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.22.ffn_gate.weight", "offset": 1008654336, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 1023277056, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 1037899776, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 1037905920, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 1037906944, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 1038324736, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.22.attn_q.bias", "offset": 1040831488, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 1040837632, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.22.attn_v.bias", "offset": 1043344384, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 1043345408, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 1043763200, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 1043769344, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.23.ffn_gate.weight", "offset": 1058392064, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 1073014784, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 1087637504, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 1087643648, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 1087644672, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 1088062464, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.23.attn_q.bias", "offset": 1090569216, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 1090575360, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.23.attn_v.bias", "offset": 1093082112, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 1093083136, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 1093500928, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 1093507072, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.24.ffn_gate.weight", "offset": 1108129792, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 1122752512, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 1137375232, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 1137381376, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 1137382400, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.24.attn_output.weight", "offset": 1137800192, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.24.attn_q.bias", "offset": 1140306944, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 1140313088, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.24.attn_v.bias", "offset": 1142819840, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 1142820864, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 1143238656, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 1143244800, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.25.ffn_gate.weight", "offset": 1157867520, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 1172490240, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 1187112960, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 1187119104, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 1187120128, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.25.attn_output.weight", "offset": 1187537920, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.25.attn_q.bias", "offset": 1190044672, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 1190050816, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.25.attn_v.bias", "offset": 1192557568, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 1192558592, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 1192976384, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 1192982528, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.26.ffn_gate.weight", "offset": 1207605248, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 1222227968, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 1236850688, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 1236856832, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 1236857856, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.26.attn_output.weight", "offset": 1237275648, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.26.attn_q.bias", "offset": 1239782400, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 1239788544, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.26.attn_v.bias", "offset": 1242295296, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 1242296320, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 1242714112, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 1242720256, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.27.ffn_gate.weight", "offset": 1257342976, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 1271965696, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 1286588416, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 1286594560, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 1286595584, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.27.attn_output.weight", "offset": 1287013376, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.27.attn_q.bias", "offset": 1289520128, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 1289526272, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.27.attn_v.bias", "offset": 1292033024, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 1292034048, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 1292451840, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "\u0000#\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0000\u0000\u0000\b\u0000\u0000\u0000\u0000", "offset": 1292457984, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.3.ffn_gate.weight", "offset": 1307080704, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 1321703424, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 1336326144, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1336332288, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1336333312, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 1336751104, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.3.attn_q.bias", "offset": 1339257856, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1339264000, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.3.attn_v.bias", "offset": 1341770752, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1341771776, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 1342189568, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1342195712, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.4.ffn_gate.weight", "offset": 1356818432, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 1371441152, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 1386063872, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1386070016, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1386071040, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 1386488832, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.4.attn_q.bias", "offset": 1388995584, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1389001728, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.4.attn_v.bias", "offset": 1391508480, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1391509504, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 1391927296, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1391933440, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.5.ffn_gate.weight", "offset": 1406556160, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 1421178880, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 1435801600, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1435807744, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1435808768, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 1436226560, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.5.attn_q.bias", "offset": 1438733312, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1438739456, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.5.attn_v.bias", "offset": 1441246208, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1441247232, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 1441665024, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1441671168, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.6.ffn_gate.weight", "offset": 1456293888, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 1470916608, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 1485539328, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 1485545472, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1485546496, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 1485964288, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.6.attn_q.bias", "offset": 1488471040, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1488477184, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.6.attn_v.bias", "offset": 1490983936, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1490984960, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 1491402752, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1491408896, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.7.ffn_gate.weight", "offset": 1506031616, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 1520654336, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 1535277056, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1535283200, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1535284224, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 1535702016, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.7.attn_q.bias", "offset": 1538208768, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1538214912, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.7.attn_v.bias", "offset": 1540721664, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1540722688, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 1541140480, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1541146624, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.8.ffn_gate.weight", "offset": 1555769344, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 1570392064, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 1585014784, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1585020928, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1585021952, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 1585439744, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.8.attn_q.bias", "offset": 1587946496, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1587952640, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.8.attn_v.bias", "offset": 1590459392, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1590460416, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 1590878208, "shape": [ 1536 ], "size": 417792, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1590884352, "shape": [ 8960, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.9.ffn_gate.weight", "offset": 1605507072, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 1620129792, "shape": [ 1536, 8960 ], "size": 14622720, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 1634752512, "shape": [ 1536 ], "size": 14622720, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1634758656, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1634759680, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 1635177472, "shape": [ 1536, 1536 ], "size": 417792, "type": 8 }, { "name": "blk.9.attn_q.bias", "offset": 1637684224, "shape": [ 1536 ], "size": 2506752, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1637690368, "shape": [ 1536, 1536 ], "size": 6144, "type": 8 }, { "name": "blk.9.attn_v.bias", "offset": 1640197120, "shape": [ 256 ], "size": 2506752, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1640198144, "shape": [ 1536, 256 ], "size": 1024, "type": 8 }, { "name": "output_norm.weight", "offset": 1640615936, "shape": [ 1536 ], "size": 417792, "type": 0 } ], "version": 3 }