qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 2, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 199186944, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 206928384, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 214669824, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 214675968, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 214676992, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 214898176, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.0.attn_q.bias", "offset": 216225280, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 216231424, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.0.attn_v.bias", "offset": 217558528, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 217559552, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 217780736, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 217786880, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 225528320, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 233269760, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 241011200, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 241017344, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 241018368, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 241239552, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.1.attn_q.bias", "offset": 242566656, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 242572800, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.1.attn_v.bias", "offset": 243899904, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 243900928, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 244122112, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 244128256, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 251869696, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 259611136, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 267352576, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 267358720, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 267359744, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 267580928, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.10.attn_q.bias", "offset": 268908032, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 268914176, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.10.attn_v.bias", "offset": 270241280, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 270242304, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 270463488, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 270469632, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 278211072, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 285952512, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 293693952, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 293700096, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 293701120, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 293922304, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.11.attn_q.bias", "offset": 295249408, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 295255552, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.11.attn_v.bias", "offset": 296582656, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 296583680, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 296804864, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 296811008, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 304552448, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 312293888, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 320035328, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 320041472, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 320042496, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 320263680, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.12.attn_q.bias", "offset": 321590784, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 321596928, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.12.attn_v.bias", "offset": 322924032, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 322925056, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 323146240, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 323152384, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 330893824, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 338635264, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 346376704, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 346382848, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 346383872, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 346605056, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.13.attn_q.bias", "offset": 347932160, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 347938304, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.13.attn_v.bias", "offset": 349265408, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 349266432, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 349487616, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 349493760, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 357235200, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 364976640, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 372718080, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 372724224, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 372725248, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 372946432, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.14.attn_q.bias", "offset": 374273536, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 374279680, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.14.attn_v.bias", "offset": 375606784, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 375607808, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 375828992, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 375835136, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 383576576, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 391318016, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 399059456, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 399065600, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 399066624, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 399287808, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.15.attn_q.bias", "offset": 400614912, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 400621056, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.15.attn_v.bias", "offset": 401948160, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 401949184, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 402170368, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 402176512, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 409917952, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 417659392, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 425400832, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 425406976, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 425408000, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 425629184, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.16.attn_q.bias", "offset": 426956288, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 426962432, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.16.attn_v.bias", "offset": 428289536, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 428290560, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 428511744, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 428517888, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 436259328, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 444000768, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 451742208, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 451748352, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 451749376, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 451970560, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.17.attn_q.bias", "offset": 453297664, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 453303808, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.17.attn_v.bias", "offset": 454630912, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 454631936, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 454853120, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 454859264, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 462600704, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 470342144, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 478083584, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 478089728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 478090752, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 478311936, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.18.attn_q.bias", "offset": 479639040, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 479645184, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.18.attn_v.bias", "offset": 480972288, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 480973312, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 481194496, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 481200640, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 488942080, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 496683520, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 504424960, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 504431104, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 504432128, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 504653312, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.19.attn_q.bias", "offset": 505980416, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 505986560, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.19.attn_v.bias", "offset": 507313664, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 507314688, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 507535872, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 507542016, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 515283456, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 523024896, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 530766336, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 530772480, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 530773504, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 530994688, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.2.attn_q.bias", "offset": 532321792, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 532327936, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.2.attn_v.bias", "offset": 533655040, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 533656064, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 533877248, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 533883392, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 541624832, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 549366272, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 557107712, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 557113856, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 557114880, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 557336064, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.20.attn_q.bias", "offset": 558663168, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 558669312, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.20.attn_v.bias", "offset": 559996416, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 559997440, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 560218624, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 560224768, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 567966208, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 575707648, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 583449088, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 583455232, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 583456256, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 583677440, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.21.attn_q.bias", "offset": 585004544, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 585010688, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.21.attn_v.bias", "offset": 586337792, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 586338816, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 586560000, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 586566144, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 594307584, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 602049024, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 609790464, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 609796608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 609797632, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 610018816, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.22.attn_q.bias", "offset": 611345920, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 611352064, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.22.attn_v.bias", "offset": 612679168, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 612680192, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 612901376, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 612907520, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 620648960, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 628390400, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 636131840, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 636137984, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 636139008, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 636360192, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.23.attn_q.bias", "offset": 637687296, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 637693440, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.23.attn_v.bias", "offset": 639020544, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 639021568, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 639242752, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 639248896, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.24.ffn_gate.weight", "offset": 646990336, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 654731776, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 662473216, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 662479360, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 662480384, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.24.attn_output.weight", "offset": 662701568, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.24.attn_q.bias", "offset": 664028672, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 664034816, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.24.attn_v.bias", "offset": 665361920, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 665362944, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 665584128, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 665590272, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.25.ffn_gate.weight", "offset": 673331712, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 681073152, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 688814592, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 688820736, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 688821760, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.25.attn_output.weight", "offset": 689042944, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.25.attn_q.bias", "offset": 690370048, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 690376192, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.25.attn_v.bias", "offset": 691703296, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 691704320, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 691925504, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 691931648, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.26.ffn_gate.weight", "offset": 699673088, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 707414528, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 715155968, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 715162112, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 715163136, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.26.attn_output.weight", "offset": 715384320, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.26.attn_q.bias", "offset": 716711424, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 716717568, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.26.attn_v.bias", "offset": 718044672, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 718045696, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 718266880, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 718273024, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.27.ffn_gate.weight", "offset": 726014464, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 733755904, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 741497344, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 741503488, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 741504512, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.27.attn_output.weight", "offset": 741725696, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.27.attn_q.bias", "offset": 743052800, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 743058944, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.27.attn_v.bias", "offset": 744386048, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 744387072, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 744608256, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 744614400, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 752355840, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 760097280, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 767838720, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 767844864, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 767845888, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 768067072, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.3.attn_q.bias", "offset": 769394176, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 769400320, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.3.attn_v.bias", "offset": 770727424, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 770728448, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 770949632, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 770955776, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 778697216, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 786438656, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 794180096, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 794186240, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 794187264, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 794408448, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.4.attn_q.bias", "offset": 795735552, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 795741696, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.4.attn_v.bias", "offset": 797068800, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 797069824, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 797291008, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 797297152, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 805038592, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 812780032, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 820521472, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 820527616, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 820528640, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 820749824, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.5.attn_q.bias", "offset": 822076928, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 822083072, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.5.attn_v.bias", "offset": 823410176, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 823411200, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 823632384, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 823638528, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 831379968, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 839121408, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 846862848, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 846868992, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 846870016, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 847091200, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.6.attn_q.bias", "offset": 848418304, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 848424448, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.6.attn_v.bias", "offset": 849751552, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 849752576, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 849973760, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 849979904, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 857721344, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 865462784, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 873204224, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 873210368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 873211392, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 873432576, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.7.attn_q.bias", "offset": 874759680, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 874765824, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.7.attn_v.bias", "offset": 876092928, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 876093952, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 876315136, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 876321280, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 884062720, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 891804160, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 899545600, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 899551744, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 899552768, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 899773952, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.8.attn_q.bias", "offset": 901101056, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 901107200, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.8.attn_v.bias", "offset": 902434304, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 902435328, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 902656512, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 902662656, "shape": [ 8960, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 910404096, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 918145536, "shape": [ 1536, 8960 ], "size": 7741440, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 925886976, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 925893120, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 925894144, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 926115328, "shape": [ 1536, 1536 ], "size": 221184, "type": 2 }, { "name": "blk.9.attn_q.bias", "offset": 927442432, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 927448576, "shape": [ 1536, 1536 ], "size": 6144, "type": 2 }, { "name": "blk.9.attn_v.bias", "offset": 928775680, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 928776704, "shape": [ 1536, 256 ], "size": 1024, "type": 2 }, { "name": "output_norm.weight", "offset": 928997888, "shape": [ 1536 ], "size": 221184, "type": 0 } ], "version": 3 }