qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 11, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 197359104, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 203272704, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 209186304, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 209192448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 209193472, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 209362432, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.0.attn_q.bias", "offset": 210376192, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 210382336, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 211396096, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 211397120, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 211566080, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 211572224, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 217485824, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 223399424, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 229313024, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 229319168, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 229320192, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 229489152, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.1.attn_q.bias", "offset": 230502912, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 230509056, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 231522816, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 231523840, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 231692800, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 231698944, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 237612544, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 243526144, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 249439744, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 249445888, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 249446912, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 249615872, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.10.attn_q.bias", "offset": 250629632, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 250635776, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 251649536, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 251650560, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 251819520, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 251825664, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "\u0000\u0000\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0000\u0000\u0000\u000b\u0000\u0000\u0000\u000065\u0016\u0000\u0000", "offset": 257739264, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 263652864, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 269566464, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 269572608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 269573632, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 269742592, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.11.attn_q.bias", "offset": 270756352, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 270762496, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 271776256, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 271777280, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 271946240, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 271952384, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 277865984, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 283779584, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 289693184, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 289699328, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 289700352, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 289869312, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.12.attn_q.bias", "offset": 290883072, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 290889216, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 291902976, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 291904000, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 292072960, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 292079104, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 297992704, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 303906304, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 309819904, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 309826048, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 309827072, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 309996032, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.13.attn_q.bias", "offset": 311009792, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 311015936, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 312029696, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 312030720, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 312199680, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 312205824, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 318119424, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 324033024, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 329946624, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 329952768, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 329953792, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 330122752, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.14.attn_q.bias", "offset": 331136512, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 331142656, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 332156416, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 332157440, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 332326400, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 332332544, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 338246144, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 344159744, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 350073344, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 350079488, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 350080512, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 350249472, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.15.attn_q.bias", "offset": 351263232, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 351269376, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 352283136, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 352284160, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 352453120, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 352459264, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 358372864, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 364286464, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 370200064, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 370206208, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 370207232, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 370376192, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.16.attn_q.bias", "offset": 371389952, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 371396096, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 372409856, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 372410880, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 372579840, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 372585984, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 378499584, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 384413184, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 390326784, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 390332928, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 390333952, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 390502912, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.17.attn_q.bias", "offset": 391516672, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 391522816, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 392536576, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 392537600, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 392706560, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 392712704, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 398626304, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 404539904, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 410453504, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 410459648, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 410460672, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 410629632, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.18.attn_q.bias", "offset": 411643392, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 411649536, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 412663296, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 412664320, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 412833280, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 412839424, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 418753024, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 424666624, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 430580224, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 430586368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 430587392, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 430756352, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.19.attn_q.bias", "offset": 431770112, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 431776256, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 432790016, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 432791040, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 432960000, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 432966144, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 438879744, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 444793344, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 450706944, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 450713088, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 450714112, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 450883072, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.2.attn_q.bias", "offset": 451896832, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 451902976, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 452916736, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 452917760, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 453086720, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 453092864, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 459006464, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 464920064, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 470833664, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 470839808, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 470840832, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 471009792, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.20.attn_q.bias", "offset": 472023552, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 472029696, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 473043456, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 473044480, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 473213440, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 473219584, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 479133184, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 485046784, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 490960384, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 490966528, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 490967552, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 491136512, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.21.attn_q.bias", "offset": 492150272, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 492156416, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 493170176, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 493171200, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.22.attn_norm.weight", "offset": 493340160, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 493346304, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 499259904, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 505173504, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 511087104, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 511093248, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 511094272, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 511263232, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.22.attn_q.bias", "offset": 512276992, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 512283136, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 513296896, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 513297920, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 513466880, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 513473024, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 519386624, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 525300224, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 531213824, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 531219968, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 531220992, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 531389952, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.23.attn_q.bias", "offset": 532403712, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 532409856, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 533423616, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 533424640, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 533593600, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 533599744, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.ffn_gate.weight", "offset": 539513344, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 545426944, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 551340544, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 551346688, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 551347712, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 551516672, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.24.attn_q.bias", "offset": 552530432, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 552536576, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 553550336, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 553551360, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 553720320, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 553726464, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.ffn_gate.weight", "offset": 559640064, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 565553664, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 571467264, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 571473408, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 571474432, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 571643392, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.25.attn_q.bias", "offset": 572657152, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 572663296, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 573677056, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 573678080, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 573847040, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 573853184, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.ffn_gate.weight", "offset": 579766784, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 585680384, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 591593984, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 591600128, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 591601152, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 591770112, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.26.attn_q.bias", "offset": 592783872, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 592790016, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 593803776, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 593804800, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 593973760, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 593979904, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.ffn_gate.weight", "offset": 599893504, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 605807104, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 611720704, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 611726848, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 611727872, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 611896832, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.27.attn_q.bias", "offset": 612910592, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 612916736, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 613930496, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 613931520, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 614100480, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000.\ufffd+\u0000\u0000\u0000\u0000\u0015\u0000\u0000\u0000\u0000\u0000\u0000\u0000bl", "offset": 614106624, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 620020224, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 625933824, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 631847424, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 631853568, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 631854592, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 632023552, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.3.attn_q.bias", "offset": 633037312, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 633043456, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 634057216, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 634058240, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 634227200, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 634233344, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 640146944, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 646060544, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 651974144, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 651980288, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 651981312, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 652150272, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.4.attn_q.bias", "offset": 653164032, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 653170176, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 654183936, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 654184960, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 654353920, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 654360064, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 660273664, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 666187264, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 672100864, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 672107008, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 672108032, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 672276992, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.5.attn_q.bias", "offset": 673290752, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 673296896, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 674310656, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 674311680, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 674480640, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 674486784, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 680400384, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 686313984, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 692227584, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 692233728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 692234752, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 692403712, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.6.attn_q.bias", "offset": 693417472, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 693423616, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 694437376, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 694438400, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.7.attn_norm.weight", "offset": 694607360, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 694613504, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 700527104, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 706440704, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 712354304, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 712360448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 712361472, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 712530432, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.7.attn_q.bias", "offset": 713544192, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 713550336, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 714564096, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 714565120, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 714734080, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 714740224, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 720653824, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 726567424, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 732481024, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 732487168, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 732488192, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 732657152, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.8.attn_q.bias", "offset": 733670912, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 733677056, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 734690816, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 734691840, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 734860800, "shape": [ 1536 ], "size": 168960, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 734866944, "shape": [ 8960, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 740780544, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 746694144, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 752607744, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 752613888, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 752614912, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 752783872, "shape": [ 1536, 1536 ], "size": 168960, "type": 11 }, { "name": "blk.9.attn_q.bias", "offset": 753797632, "shape": [ 1536 ], "size": 1013760, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 753803776, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 754817536, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 754818560, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "output_norm.weight", "offset": 754987520, "shape": [ 1536 ], "size": 168960, "type": 0 } ], "version": 3 }