qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 13, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 200907264, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 206820864, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 212734464, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 212740608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 212741632, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 212910592, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 214532608, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 214538752, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 215552512, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 215553536, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 215823872, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 215830016, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 225291776, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 231205376, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 237118976, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 237125120, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 237126144, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 237295104, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 238917120, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 238923264, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 239937024, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 239938048, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 240208384, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 240214528, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 249676288, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 255589888, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 261503488, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 261509632, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 261510656, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 261679616, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 263301632, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 263307776, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 264321536, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 264322560, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 264592896, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 264599040, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 274060800, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 279974400, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 285888000, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 285894144, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 285895168, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 286064128, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 287686144, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 287692288, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 288706048, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 288707072, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 288977408, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 288983552, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 298445312, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 304358912, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 310272512, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 310278656, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 310279680, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 310448640, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 312070656, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 312076800, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 313090560, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 313091584, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 313361920, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 313368064, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 322829824, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 328743424, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 334657024, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 334663168, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 334664192, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 334833152, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 336455168, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 336461312, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 337475072, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 337476096, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 337746432, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 337752576, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 347214336, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 353127936, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 359041536, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 359047680, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 359048704, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 359217664, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 360839680, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 360845824, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 361859584, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 361860608, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 362130944, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 362137088, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 371598848, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 377512448, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 383426048, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 383432192, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 383433216, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 383602176, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 385224192, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 385230336, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 386244096, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 386245120, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 386515456, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 386521600, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 395983360, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 401896960, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 407810560, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 407816704, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 407817728, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 407986688, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 409608704, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 409614848, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 410628608, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 410629632, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 410899968, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 410906112, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 420367872, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 426281472, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 432195072, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 432201216, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 432202240, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 432371200, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 433993216, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 433999360, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 435013120, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 435014144, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 435284480, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 435290624, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 444752384, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 450665984, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 456579584, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 456585728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 456586752, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 456755712, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 458377728, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 458383872, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 459397632, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 459398656, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 459668992, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 459675136, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 469136896, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 475050496, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 480964096, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 480970240, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 480971264, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 481140224, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 482762240, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 482768384, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 483782144, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 483783168, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 484053504, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 484059648, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 493521408, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 499435008, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 505348608, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 505354752, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 505355776, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 505524736, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 507146752, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 507152896, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 508166656, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 508167680, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 508438016, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 508444160, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 517905920, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 523819520, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 529733120, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 529739264, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 529740288, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 529909248, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 531531264, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 531537408, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 532551168, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 532552192, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 532822528, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 532828672, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 542290432, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 548204032, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 554117632, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 554123776, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 554124800, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 554293760, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 555915776, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 555921920, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 556935680, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 556936704, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 557207040, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 557213184, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 566674944, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 572588544, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 578502144, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 578508288, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 578509312, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 578678272, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 580300288, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 580306432, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 581320192, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 581321216, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 581591552, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 581597696, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 591059456, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 596973056, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 602886656, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 602892800, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 602893824, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 603062784, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 604684800, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 604690944, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 605704704, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 605705728, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 605976064, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 605982208, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 615443968, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 621357568, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 627271168, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 627277312, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 627278336, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 627447296, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 629069312, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 629075456, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 630089216, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 630090240, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 630360576, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 630366720, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 639828480, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 645742080, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 651655680, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 651661824, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 651662848, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 651831808, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 653453824, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 653459968, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 654473728, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 654474752, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 654745088, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 654751232, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 664212992, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 670126592, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 676040192, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 676046336, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 676047360, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 676216320, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 677838336, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 677844480, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 678858240, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 678859264, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 679129600, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 679135744, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 688597504, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 694511104, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 700424704, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 700430848, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 700431872, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 700600832, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 702222848, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 702228992, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 703242752, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 703243776, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 703514112, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 703520256, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 712982016, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 718895616, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 724809216, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 724815360, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 724816384, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 724985344, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 726607360, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 726613504, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 727627264, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 727628288, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 727898624, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 727904768, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 737366528, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 743280128, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 749193728, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 749199872, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 749200896, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 749369856, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 750991872, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 750998016, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 752011776, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 752012800, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 752283136, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 752289280, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 761751040, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 767664640, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 773578240, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 773584384, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 773585408, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 773754368, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 775376384, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 775382528, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 776396288, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 776397312, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 776667648, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 776673792, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 786135552, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 792049152, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 797962752, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 797968896, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 797969920, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 798138880, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 799760896, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 799767040, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 800780800, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 800781824, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 801052160, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 801058304, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 810520064, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 816433664, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 822347264, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 822353408, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 822354432, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 822523392, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 824145408, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 824151552, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 825165312, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 825166336, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 825436672, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 825442816, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 834904576, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 840818176, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 846731776, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 846737920, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 846738944, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 846907904, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 848529920, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 848536064, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 849549824, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 849550848, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 849821184, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 849827328, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 859289088, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 865202688, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 871116288, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 871122432, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 871123456, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 871292416, "shape": [ 1536, 1536 ], "size": 168960, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 872914432, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 872920576, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 873934336, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 873935360, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "output_norm.weight", "offset": 874205696, "shape": [ 1536 ], "size": 270336, "type": 0 } ], "version": 3 }