qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 17, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 148221696, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 151490304, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 154758912, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 154762496, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 154763008, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 154849024, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 155451136, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 155454720, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.0.attn_v.bias", "offset": 156056832, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 156057344, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 156179200, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 156182784, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 159757824, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 163026432, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 166295040, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 166298624, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 166299136, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 166385152, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 166987264, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 166990848, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.1.attn_v.bias", "offset": 167592960, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 167593472, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 167715328, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 167718912, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 171293952, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 174562560, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 177831168, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 177834752, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 177835264, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 177921280, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 178523392, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 178526976, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.10.attn_v.bias", "offset": 179129088, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 179129600, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 179251456, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 179255040, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 182251264, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 185519872, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 188788480, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 188792064, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 188792576, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 188878592, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 189480704, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 189484288, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.11.attn_v.bias", "offset": 190086400, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 190086912, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 190172928, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 190176512, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 193172736, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 196441344, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 199709952, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 199713536, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 199714048, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 199800064, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 200402176, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 200405760, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.12.attn_v.bias", "offset": 201007872, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 201008384, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 201094400, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 201097984, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 204673024, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 207941632, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 211210240, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 211213824, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 211214336, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 211300352, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 211902464, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 211906048, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.13.attn_v.bias", "offset": 212508160, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 212508672, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 212630528, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 212634112, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 215630336, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 218898944, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 222167552, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 222171136, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 222171648, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 222257664, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 222859776, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 222863360, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.14.attn_v.bias", "offset": 223465472, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 223465984, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 223552000, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 223555584, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 226551808, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 229820416, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 233089024, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 233092608, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 233093120, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 233179136, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 233781248, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 233784832, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.15.attn_v.bias", "offset": 234386944, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 234387456, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 234473472, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 234477056, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 238052096, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 241320704, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 244589312, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 244592896, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 244593408, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 244679424, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 245281536, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 245285120, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.16.attn_v.bias", "offset": 245887232, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 245887744, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 246009600, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 246013184, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 249009408, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 252278016, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 255546624, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 255550208, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 255550720, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 255636736, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 256238848, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 256242432, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.17.attn_v.bias", "offset": 256844544, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 256845056, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 256931072, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 256934656, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 259930880, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 263199488, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 266468096, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 266471680, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 266472192, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 266558208, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 267160320, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 267163904, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.18.attn_v.bias", "offset": 267766016, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 267766528, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 267852544, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 267856128, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 271431168, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 274699776, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 277968384, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 277971968, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 277972480, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 278058496, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 278660608, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 278664192, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.19.attn_v.bias", "offset": 279266304, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 279266816, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 279388672, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 279392256, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 282388480, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 285657088, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 288925696, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 288929280, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 288929792, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 289015808, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 289617920, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 289621504, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.2.attn_v.bias", "offset": 290223616, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 290224128, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 290310144, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 290313728, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 293309952, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 296578560, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 299847168, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 299850752, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 299851264, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 299937280, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 300539392, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 300542976, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.20.attn_v.bias", "offset": 301145088, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 301145600, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 301231616, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 301235200, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 304810240, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 308078848, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 311347456, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 311351040, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 311351552, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 311437568, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 312039680, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 312043264, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.21.attn_v.bias", "offset": 312645376, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 312645888, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 312767744, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 312771328, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 315767552, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 319036160, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 322304768, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 322308352, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 322308864, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 322394880, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 322996992, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 323000576, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.22.attn_v.bias", "offset": 323602688, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 323603200, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 323689216, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 323692800, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 326689024, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 329957632, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 333226240, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 333229824, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 333230336, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 333316352, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 333918464, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 333922048, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.23.attn_v.bias", "offset": 334524160, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 334524672, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 334610688, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 334614272, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 338189312, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 341457920, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 344726528, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 344730112, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 344730624, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 344816640, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 345418752, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 345422336, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.3.attn_v.bias", "offset": 346024448, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 346024960, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 346146816, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 346150400, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 349146624, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 352415232, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 355683840, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 355687424, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 355687936, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 355773952, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 356376064, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 356379648, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.4.attn_v.bias", "offset": 356981760, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 356982272, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 357068288, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 357071872, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 360068096, "shape": [ 896, 4864 ], "size": 2996224, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 363336704, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 366605312, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 366608896, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 366609408, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 366695424, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 367297536, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 367301120, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.5.attn_v.bias", "offset": 367903232, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 367903744, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 367989760, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 367993344, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 371568384, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 374836992, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 378105600, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 378109184, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 378109696, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 378195712, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 378797824, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 378801408, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.6.attn_v.bias", "offset": 379403520, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 379404032, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 379525888, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 379529472, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 383104512, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 386373120, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 389641728, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 389645312, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 389645824, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 389731840, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 390333952, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 390337536, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.7.attn_v.bias", "offset": 390939648, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 390940160, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 391062016, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 391065600, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 394640640, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 397909248, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 401177856, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 401181440, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 401181952, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 401267968, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 401870080, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 401873664, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.8.attn_v.bias", "offset": 402475776, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 402476288, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 402598144, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 402601728, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 406176768, "shape": [ 896, 4864 ], "size": 3575040, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 409445376, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 412713984, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 412717568, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 412718080, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 412804096, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 413406208, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 413409792, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.9.attn_v.bias", "offset": 414011904, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 414012416, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "output_norm.weight", "offset": 414134272, "shape": [ 896 ], "size": 121856, "type": 0 } ], "version": 3 }