qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 16, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 147642880, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 150911488, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 154180096, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 154183680, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 154184192, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 154270208, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 154872320, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 154875904, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.0.attn_v.bias", "offset": 155478016, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 155478528, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 155564544, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 155568128, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 158564352, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 161832960, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 165101568, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 165105152, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 165105664, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 165191680, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 165793792, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 165797376, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.1.attn_v.bias", "offset": 166399488, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 166400000, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 166486016, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 166489600, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 169485824, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 172754432, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 176023040, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 176026624, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 176027136, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 176113152, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 176715264, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 176718848, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.10.attn_v.bias", "offset": 177320960, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 177321472, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 177407488, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 177411072, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "\u0000\u0000\u0000\u0000\ufffd\u0003\u0000\u0000\u0000\u0000\u0000\u0000\r\u0000\u0000\u0000\u0000\ufffdz\u000e\u0000\u0000", "offset": 180407296, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 183675904, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 186944512, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 186948096, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 186948608, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 187034624, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 187636736, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 187640320, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.11.attn_v.bias", "offset": 188242432, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 188242944, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 188328960, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 188332544, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 191328768, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 194597376, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 197865984, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 197869568, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 197870080, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 197956096, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 198558208, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 198561792, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.12.attn_v.bias", "offset": 199163904, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 199164416, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 199250432, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 199254016, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 202250240, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 205518848, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 208787456, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 208791040, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 208791552, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 208877568, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 209479680, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 209483264, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.13.attn_v.bias", "offset": 210085376, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 210085888, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 210171904, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 210175488, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 213171712, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 216440320, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 219708928, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 219712512, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 219713024, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 219799040, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 220401152, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 220404736, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.14.attn_v.bias", "offset": 221006848, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 221007360, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 221093376, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 221096960, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 224093184, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 227361792, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 230630400, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 230633984, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 230634496, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 230720512, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 231322624, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 231326208, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.15.attn_v.bias", "offset": 231928320, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 231928832, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 232014848, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 232018432, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 235014656, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 238283264, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 241551872, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 241555456, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 241555968, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 241641984, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 242244096, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 242247680, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.16.attn_v.bias", "offset": 242849792, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 242850304, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 242936320, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 242939904, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 245936128, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 249204736, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 252473344, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 252476928, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 252477440, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 252563456, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 253165568, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 253169152, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.17.attn_v.bias", "offset": 253771264, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 253771776, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 253857792, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 253861376, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 256857600, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 260126208, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 263394816, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 263398400, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 263398912, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 263484928, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 264087040, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 264090624, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.18.attn_v.bias", "offset": 264692736, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 264693248, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 264779264, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 264782848, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 267779072, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 271047680, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 274316288, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 274319872, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 274320384, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 274406400, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 275008512, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 275012096, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.19.attn_v.bias", "offset": 275614208, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 275614720, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 275700736, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 275704320, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 278700544, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 281969152, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 285237760, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 285241344, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 285241856, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 285327872, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 285929984, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 285933568, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.2.attn_v.bias", "offset": 286535680, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 286536192, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 286622208, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 286625792, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 289622016, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 292890624, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 296159232, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 296162816, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 296163328, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 296249344, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 296851456, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 296855040, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.20.attn_v.bias", "offset": 297457152, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 297457664, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 297543680, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 297547264, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 300543488, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 303812096, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 307080704, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 307084288, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 307084800, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 307170816, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 307772928, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 307776512, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.21.attn_v.bias", "offset": 308378624, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 308379136, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 308465152, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 308468736, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 311464960, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 314733568, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 318002176, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 318005760, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 318006272, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 318092288, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 318694400, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 318697984, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.22.attn_v.bias", "offset": 319300096, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 319300608, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 319386624, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 319390208, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 322386432, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 325655040, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 328923648, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 328927232, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 328927744, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 329013760, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 329615872, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 329619456, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.23.attn_v.bias", "offset": 330221568, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 330222080, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 330308096, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 330311680, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 333307904, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 336576512, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 339845120, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 339848704, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 339849216, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 339935232, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 340537344, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 340540928, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.3.attn_v.bias", "offset": 341143040, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 341143552, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 341229568, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 341233152, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 344229376, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 347497984, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 350766592, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 350770176, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 350770688, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 350856704, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 351458816, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 351462400, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.4.attn_v.bias", "offset": 352064512, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 352065024, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 352151040, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 352154624, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 355150848, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 358419456, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 361688064, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 361691648, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 361692160, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 361778176, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 362380288, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 362383872, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.5.attn_v.bias", "offset": 362985984, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 362986496, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 363072512, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 363076096, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 366072320, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 369340928, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 372609536, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 372613120, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 372613632, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 372699648, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 373301760, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 373305344, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.6.attn_v.bias", "offset": 373907456, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 373907968, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 373993984, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 373997568, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 376993792, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 380262400, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 383531008, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 383534592, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 383535104, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 383621120, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 384223232, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 384226816, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.7.attn_v.bias", "offset": 384828928, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 384829440, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 384915456, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 384919040, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 387915264, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 391183872, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 394452480, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 394456064, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 394456576, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 394542592, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 395144704, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 395148288, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.8.attn_v.bias", "offset": 395750400, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 395750912, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 395836928, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 395840512, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 398836736, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 402105344, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 405373952, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 405377536, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 405378048, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 405464064, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 406066176, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 406069760, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.9.attn_v.bias", "offset": 406671872, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 406672384, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "output_norm.weight", "offset": 406758400, "shape": [ 896 ], "size": 86016, "type": 0 } ], "version": 3 }