qwen2

Qwen2 is a new series of large language models from Alibaba group

0.5B 1.5B 7B 72B

273.6K Pulls Updated 4 weeks ago

97 Tags

... /
model
1d7e6755e500 · 1.3GB
{ "metadata": { "general.architecture": "qwen2", "general.file_type": 18, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 202735104, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 214024704, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.0.ffn_norm.weight", "offset": 225314304, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 225320448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 225321472, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.0.attn_output.weight", "offset": 225644032, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.0.attn_q.bias", "offset": 227579392, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 227585536, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.0.attn_v.bias", "offset": 229520896, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 229521920, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 229844480, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 229850624, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 241140224, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 252429824, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 263719424, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 263725568, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 263726592, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.1.attn_output.weight", "offset": 264049152, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.1.attn_q.bias", "offset": 265984512, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 265990656, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.1.attn_v.bias", "offset": 267926016, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 267927040, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 268249600, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 268255744, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 279545344, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 290834944, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 302124544, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 302130688, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 302131712, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.10.attn_output.weight", "offset": 302454272, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.10.attn_q.bias", "offset": 304389632, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 304395776, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.10.attn_v.bias", "offset": 306331136, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 306332160, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 306654720, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 306660864, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 317950464, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 329240064, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 340529664, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 340535808, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 340536832, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.11.attn_output.weight", "offset": 340859392, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.11.attn_q.bias", "offset": 342794752, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 342800896, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.11.attn_v.bias", "offset": 344736256, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 344737280, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 345059840, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 345065984, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 356355584, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 367645184, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 378934784, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 378940928, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 378941952, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.12.attn_output.weight", "offset": 379264512, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.12.attn_q.bias", "offset": 381199872, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 381206016, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.12.attn_v.bias", "offset": 383141376, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 383142400, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 383464960, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 383471104, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 394760704, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 406050304, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 417339904, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 417346048, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 417347072, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.13.attn_output.weight", "offset": 417669632, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.13.attn_q.bias", "offset": 419604992, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 419611136, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.13.attn_v.bias", "offset": 421546496, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 421547520, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 421870080, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 421876224, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 433165824, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 444455424, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 455745024, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 455751168, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 455752192, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.14.attn_output.weight", "offset": 456074752, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.14.attn_q.bias", "offset": 458010112, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 458016256, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.14.attn_v.bias", "offset": 459951616, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 459952640, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 460275200, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 460281344, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 471570944, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 482860544, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 494150144, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 494156288, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 494157312, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.15.attn_output.weight", "offset": 494479872, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.15.attn_q.bias", "offset": 496415232, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 496421376, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.15.attn_v.bias", "offset": 498356736, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 498357760, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 498680320, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 498686464, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 509976064, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 521265664, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 532555264, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 532561408, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 532562432, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.16.attn_output.weight", "offset": 532884992, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.16.attn_q.bias", "offset": 534820352, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 534826496, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.16.attn_v.bias", "offset": 536761856, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 536762880, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 537085440, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 537091584, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 548381184, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 559670784, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 570960384, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 570966528, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 570967552, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.17.attn_output.weight", "offset": 571290112, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.17.attn_q.bias", "offset": 573225472, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 573231616, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.17.attn_v.bias", "offset": 575166976, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 575168000, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 575490560, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 575496704, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 586786304, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 598075904, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 609365504, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 609371648, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 609372672, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.18.attn_output.weight", "offset": 609695232, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.18.attn_q.bias", "offset": 611630592, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 611636736, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.18.attn_v.bias", "offset": 613572096, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 613573120, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 613895680, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 613901824, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 625191424, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 636481024, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 647770624, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 647776768, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 647777792, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.19.attn_output.weight", "offset": 648100352, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.19.attn_q.bias", "offset": 650035712, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 650041856, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.19.attn_v.bias", "offset": 651977216, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 651978240, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 652300800, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 652306944, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 663596544, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 674886144, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 686175744, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 686181888, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 686182912, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.2.attn_output.weight", "offset": 686505472, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.2.attn_q.bias", "offset": 688440832, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 688446976, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.2.attn_v.bias", "offset": 690382336, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 690383360, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 690705920, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 690712064, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 702001664, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 713291264, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 724580864, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 724587008, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 724588032, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.20.attn_output.weight", "offset": 724910592, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.20.attn_q.bias", "offset": 726845952, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 726852096, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.20.attn_v.bias", "offset": 728787456, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 728788480, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 729111040, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 729117184, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 740406784, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 751696384, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 762985984, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 762992128, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 762993152, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 763315712, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.21.attn_q.bias", "offset": 765251072, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 765257216, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.21.attn_v.bias", "offset": 767192576, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 767193600, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 767516160, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 767522304, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 778811904, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 790101504, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 801391104, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 801397248, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 801398272, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.22.attn_output.weight", "offset": 801720832, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.22.attn_q.bias", "offset": 803656192, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 803662336, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.22.attn_v.bias", "offset": 805597696, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 805598720, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 805921280, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 805927424, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 817217024, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 828506624, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 839796224, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 839802368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 839803392, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.23.attn_output.weight", "offset": 840125952, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.23.attn_q.bias", "offset": 842061312, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 842067456, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.23.attn_v.bias", "offset": 844002816, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 844003840, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 844326400, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 844332544, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 855622144, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 866911744, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 878201344, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 878207488, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 878208512, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.24.attn_output.weight", "offset": 878531072, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.24.attn_q.bias", "offset": 880466432, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 880472576, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.24.attn_v.bias", "offset": 882407936, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 882408960, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 882731520, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 882737664, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.25.ffn_gate.weight", "offset": 894027264, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 905316864, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 916606464, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 916612608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 916613632, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.25.attn_output.weight", "offset": 916936192, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.25.attn_q.bias", "offset": 918871552, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 918877696, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.25.attn_v.bias", "offset": 920813056, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 920814080, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 921136640, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 921142784, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.26.ffn_gate.weight", "offset": 932432384, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 943721984, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 955011584, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 955017728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 955018752, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.26.attn_output.weight", "offset": 955341312, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.26.attn_q.bias", "offset": 957276672, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 957282816, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.26.attn_v.bias", "offset": 959218176, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 959219200, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 959541760, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 959547904, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 970837504, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 982127104, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.27.ffn_norm.weight", "offset": 993416704, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 993422848, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 993423872, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.27.attn_output.weight", "offset": 993746432, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.27.attn_q.bias", "offset": 995681792, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 995687936, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.27.attn_v.bias", "offset": 997623296, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 997624320, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 997946880, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 997953024, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 1009242624, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 1020532224, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 1031821824, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 1031827968, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 1031828992, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.3.attn_output.weight", "offset": 1032151552, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.3.attn_q.bias", "offset": 1034086912, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 1034093056, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.3.attn_v.bias", "offset": 1036028416, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 1036029440, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1036352000, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1036358144, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 1047647744, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1058937344, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 1070226944, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 1070233088, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 1070234112, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.4.attn_output.weight", "offset": 1070556672, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.4.attn_q.bias", "offset": 1072492032, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 1072498176, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.4.attn_v.bias", "offset": 1074433536, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 1074434560, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1074757120, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1074763264, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 1086052864, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 1097342464, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 1108632064, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 1108638208, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 1108639232, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.5.attn_output.weight", "offset": 1108961792, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.5.attn_q.bias", "offset": 1110897152, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 1110903296, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.5.attn_v.bias", "offset": 1112838656, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 1112839680, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 1113162240, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1113168384, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 1124457984, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 1135747584, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 1147037184, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 1147043328, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1147044352, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 1147366912, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.6.attn_q.bias", "offset": 1149302272, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1149308416, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.6.attn_v.bias", "offset": 1151243776, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1151244800, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 1151567360, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1151573504, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 1162863104, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 1174152704, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 1185442304, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1185448448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1185449472, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.7.attn_output.weight", "offset": 1185772032, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.7.attn_q.bias", "offset": 1187707392, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1187713536, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.7.attn_v.bias", "offset": 1189648896, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1189649920, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 1189972480, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1189978624, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 1201268224, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 1212557824, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 1223847424, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1223853568, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1223854592, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.8.attn_output.weight", "offset": 1224177152, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.8.attn_q.bias", "offset": 1226112512, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1226118656, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.8.attn_v.bias", "offset": 1228054016, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1228055040, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 1228377600, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1228383744, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 1239673344, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 1250962944, "shape": [ 1536, 8960 ], "size": 11289600, "type": 14 }, { "name": "blk.9.ffn_norm.weight", "offset": 1262252544, "shape": [ 1536 ], "size": 11289600, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1262258688, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1262259712, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.9.attn_output.weight", "offset": 1262582272, "shape": [ 1536, 1536 ], "size": 322560, "type": 14 }, { "name": "blk.9.attn_q.bias", "offset": 1264517632, "shape": [ 1536 ], "size": 1935360, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1264523776, "shape": [ 1536, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.9.attn_v.bias", "offset": 1266459136, "shape": [ 256 ], "size": 1935360, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1266460160, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "output_norm.weight", "offset": 1266782720, "shape": [ 1536 ], "size": 322560, "type": 0 } ], "version": 3 }