qwen2

Qwen2 is a new series of large language models from Alibaba group

0.5B 1.5B 7B 72B

273.5K Pulls Updated 4 weeks ago

97 Tags

... /
model
9d9344b43f5a · 1.1GB
{ "metadata": { "general.architecture": "qwen2", "general.file_type": 17, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 202735104, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 212196864, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 221658624, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 221664768, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 221665792, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 221936128, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 223558144, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 223564288, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.0.attn_v.bias", "offset": 225186304, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 225187328, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 225509888, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 225516032, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 236805632, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 246267392, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 255729152, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 255735296, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 255736320, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 256006656, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 257628672, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 257634816, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.1.attn_v.bias", "offset": 259256832, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 259257856, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 259580416, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 259586560, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 270876160, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 280337920, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 289799680, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 289805824, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 289806848, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 290077184, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 291699200, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 291705344, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.10.attn_v.bias", "offset": 293327360, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 293328384, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 293650944, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 293657088, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 303118848, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 312580608, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 322042368, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 322048512, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 322049536, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 322319872, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 323941888, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 323948032, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.11.attn_v.bias", "offset": 325570048, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 325571072, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 325841408, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 325847552, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 335309312, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 344771072, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 354232832, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 354238976, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 354240000, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 354510336, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 356132352, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 356138496, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.12.attn_v.bias", "offset": 357760512, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 357761536, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 358031872, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 358038016, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 369327616, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 378789376, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 388251136, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 388257280, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 388258304, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 388528640, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 390150656, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 390156800, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.13.attn_v.bias", "offset": 391778816, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 391779840, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 392102400, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 392108544, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 401570304, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 411032064, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 420493824, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 420499968, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 420500992, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 420771328, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 422393344, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 422399488, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.14.attn_v.bias", "offset": 424021504, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 424022528, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 424292864, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 424299008, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 433760768, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 443222528, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 452684288, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 452690432, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 452691456, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 452961792, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 454583808, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 454589952, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.15.attn_v.bias", "offset": 456211968, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 456212992, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 456483328, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 456489472, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 467779072, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 477240832, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 486702592, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 486708736, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 486709760, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 486980096, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 488602112, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 488608256, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.16.attn_v.bias", "offset": 490230272, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 490231296, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 490553856, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 490560000, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 500021760, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 509483520, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 518945280, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 518951424, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 518952448, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 519222784, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 520844800, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 520850944, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.17.attn_v.bias", "offset": 522472960, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 522473984, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 522744320, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 522750464, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 532212224, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 541673984, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 551135744, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 551141888, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 551142912, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 551413248, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 553035264, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 553041408, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.18.attn_v.bias", "offset": 554663424, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 554664448, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 554934784, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 554940928, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 566230528, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 575692288, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 585154048, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 585160192, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 585161216, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 585431552, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 587053568, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 587059712, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.19.attn_v.bias", "offset": 588681728, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 588682752, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 589005312, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 589011456, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 598473216, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 607934976, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 617396736, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 617402880, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 617403904, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 617674240, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 619296256, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 619302400, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.2.attn_v.bias", "offset": 620924416, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 620925440, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 621195776, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 621201920, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 630663680, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 640125440, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 649587200, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 649593344, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 649594368, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 649864704, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 651486720, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 651492864, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.20.attn_v.bias", "offset": 653114880, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 653115904, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 653386240, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 653392384, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 664681984, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 674143744, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 683605504, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 683611648, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 683612672, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 683883008, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 685505024, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 685511168, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.21.attn_v.bias", "offset": 687133184, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 687134208, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 687456768, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 687462912, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 696924672, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 706386432, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 715848192, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 715854336, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 715855360, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 716125696, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 717747712, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 717753856, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.22.attn_v.bias", "offset": 719375872, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 719376896, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 719647232, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 719653376, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 729115136, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 738576896, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 748038656, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 748044800, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 748045824, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 748316160, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 749938176, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 749944320, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.23.attn_v.bias", "offset": 751566336, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 751567360, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 751837696, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 751843840, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 763133440, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 772595200, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 782056960, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 782063104, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 782064128, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 782334464, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 783956480, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 783962624, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.24.attn_v.bias", "offset": 785584640, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 785585664, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 785908224, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 785914368, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 795376128, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 804837888, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 814299648, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 814305792, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 814306816, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 814577152, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 816199168, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 816205312, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.25.attn_v.bias", "offset": 817827328, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 817828352, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 818098688, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 818104832, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 827566592, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 837028352, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 846490112, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 846496256, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 846497280, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 846767616, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 848389632, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 848395776, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.26.attn_v.bias", "offset": 850017792, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 850018816, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 850289152, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 850295296, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 861584896, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 871046656, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 880508416, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 880514560, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 880515584, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 880785920, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 882407936, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 882414080, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.27.attn_v.bias", "offset": 884036096, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 884037120, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 884359680, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "\u0000#\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0000\u0000\u0000\r\u0000\u0000\u0000\u0000", "offset": 884365824, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 893827584, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 903289344, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 912751104, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 912757248, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 912758272, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 913028608, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 914650624, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 914656768, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.3.attn_v.bias", "offset": 916278784, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 916279808, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 916550144, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 916556288, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 926018048, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 935479808, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 944941568, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 944947712, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 944948736, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 945219072, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 946841088, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 946847232, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.4.attn_v.bias", "offset": 948469248, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 948470272, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 948740608, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 948746752, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 960036352, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 969498112, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 978959872, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 978966016, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 978967040, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 979237376, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 980859392, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 980865536, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.5.attn_v.bias", "offset": 982487552, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 982488576, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 982811136, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 982817280, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 994106880, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 1003568640, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 1013030400, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 1013036544, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 1013037568, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 1013307904, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 1014929920, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 1014936064, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.6.attn_v.bias", "offset": 1016558080, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 1016559104, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 1016881664, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1016887808, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 1028177408, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 1037639168, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 1047100928, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1047107072, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1047108096, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 1047378432, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 1049000448, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1049006592, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.7.attn_v.bias", "offset": 1050628608, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1050629632, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 1050952192, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1050958336, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 1062247936, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 1071709696, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 1081171456, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1081177600, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1081178624, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 1081448960, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 1083070976, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1083077120, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.8.attn_v.bias", "offset": 1084699136, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1084700160, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 1085022720, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1085028864, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 1096318464, "shape": [ 1536, 8960 ], "size": 11289600, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 1105780224, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 1115241984, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1115248128, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1115249152, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 1115519488, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 1117141504, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1117147648, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.9.attn_v.bias", "offset": 1118769664, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1118770688, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "output_norm.weight", "offset": 1119093248, "shape": [ 1536 ], "size": 322560, "type": 0 } ], "version": 3 }