qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 9, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.0.ffn_gate.weight", "offset": 147915264, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 151183872, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 154452480, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 154456064, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 154456576, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.0.attn_output.weight", "offset": 154542592, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 155144704, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 155148288, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.0.attn_v.bias", "offset": 155750400, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 155750912, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 155836928, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 155840512, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.1.ffn_gate.weight", "offset": 159109120, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 162377728, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 165646336, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 165649920, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 165650432, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_output.weight", "offset": 165736448, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 166338560, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 166342144, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.1.attn_v.bias", "offset": 166944256, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 166944768, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 167030784, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 167034368, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.10.ffn_gate.weight", "offset": 170302976, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 173571584, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 176840192, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 176843776, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 176844288, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_output.weight", "offset": 176930304, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 177532416, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 177536000, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.10.attn_v.bias", "offset": 178138112, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 178138624, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 178224640, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 178228224, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "\u0000\u0000\u0000\u0000\ufffd\u0003\u0000\u0000\u0000\u0000\u0000\u0000\u0007\u0000\u0000\u0000\u0000`\ufffd\u000e\u0000\u0000", "offset": 181496832, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 184765440, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 188034048, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 188037632, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 188038144, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.11.attn_output.weight", "offset": 188124160, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 188726272, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 188729856, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.11.attn_v.bias", "offset": 189331968, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 189332480, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 189418496, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 189422080, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.12.ffn_gate.weight", "offset": 192690688, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 195959296, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 199227904, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 199231488, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 199232000, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_output.weight", "offset": 199318016, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 199920128, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 199923712, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.12.attn_v.bias", "offset": 200525824, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 200526336, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 200612352, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 200615936, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.13.ffn_gate.weight", "offset": 203884544, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 207153152, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 210421760, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 210425344, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 210425856, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.13.attn_output.weight", "offset": 210511872, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 211113984, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 211117568, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.13.attn_v.bias", "offset": 211719680, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 211720192, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 211806208, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 211809792, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.14.ffn_gate.weight", "offset": 215078400, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 218347008, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 221615616, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 221619200, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 221619712, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.14.attn_output.weight", "offset": 221705728, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 222307840, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 222311424, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.14.attn_v.bias", "offset": 222913536, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 222914048, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 223000064, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 223003648, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.15.ffn_gate.weight", "offset": 226272256, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 229540864, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 232809472, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 232813056, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 232813568, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.15.attn_output.weight", "offset": 232899584, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 233501696, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 233505280, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.15.attn_v.bias", "offset": 234107392, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 234107904, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 234193920, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 234197504, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.16.ffn_gate.weight", "offset": 237466112, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 240734720, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 244003328, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 244006912, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 244007424, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.16.attn_output.weight", "offset": 244093440, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 244695552, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 244699136, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.16.attn_v.bias", "offset": 245301248, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 245301760, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 245387776, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 245391360, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.17.ffn_gate.weight", "offset": 248659968, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 251928576, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 255197184, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 255200768, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 255201280, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.17.attn_output.weight", "offset": 255287296, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 255889408, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 255892992, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.17.attn_v.bias", "offset": 256495104, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 256495616, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 256581632, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 256585216, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.18.ffn_gate.weight", "offset": 259853824, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 263122432, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 266391040, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 266394624, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 266395136, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.18.attn_output.weight", "offset": 266481152, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 267083264, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 267086848, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.18.attn_v.bias", "offset": 267688960, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 267689472, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 267775488, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 267779072, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.19.ffn_gate.weight", "offset": 271047680, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 274316288, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 277584896, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 277588480, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 277588992, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.19.attn_output.weight", "offset": 277675008, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 278277120, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 278280704, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.19.attn_v.bias", "offset": 278882816, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 278883328, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 278969344, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 278972928, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.2.ffn_gate.weight", "offset": 282241536, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 285510144, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 288778752, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 288782336, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 288782848, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.2.attn_output.weight", "offset": 288868864, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 289470976, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 289474560, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.2.attn_v.bias", "offset": 290076672, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 290077184, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 290163200, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 290166784, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.20.ffn_gate.weight", "offset": 293435392, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 296704000, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 299972608, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 299976192, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 299976704, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.20.attn_output.weight", "offset": 300062720, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 300664832, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 300668416, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.20.attn_v.bias", "offset": 301270528, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 301271040, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 301357056, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 301360640, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.21.ffn_gate.weight", "offset": 304629248, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 307897856, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 311166464, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 311170048, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 311170560, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 311256576, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 311858688, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 311862272, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.21.attn_v.bias", "offset": 312464384, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 312464896, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 312550912, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 312554496, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.22.ffn_gate.weight", "offset": 315823104, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 319091712, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 322360320, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 322363904, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 322364416, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.22.attn_output.weight", "offset": 322450432, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 323052544, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 323056128, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.22.attn_v.bias", "offset": 323658240, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 323658752, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 323744768, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 323748352, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.23.ffn_gate.weight", "offset": 327016960, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 330285568, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 333554176, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 333557760, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 333558272, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.23.attn_output.weight", "offset": 333644288, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 334246400, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 334249984, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.23.attn_v.bias", "offset": 334852096, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 334852608, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 334938624, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 334942208, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.3.ffn_gate.weight", "offset": 338210816, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 341479424, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 344748032, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 344751616, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 344752128, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.3.attn_output.weight", "offset": 344838144, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 345440256, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 345443840, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.3.attn_v.bias", "offset": 346045952, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 346046464, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 346132480, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 346136064, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.4.ffn_gate.weight", "offset": 349404672, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 352673280, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 355941888, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 355945472, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 355945984, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.4.attn_output.weight", "offset": 356032000, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 356634112, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 356637696, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.4.attn_v.bias", "offset": 357239808, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 357240320, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 357326336, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 357329920, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.5.ffn_gate.weight", "offset": 360598528, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 363867136, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 367135744, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 367139328, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 367139840, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.5.attn_output.weight", "offset": 367225856, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 367827968, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 367831552, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.5.attn_v.bias", "offset": 368433664, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 368434176, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 368520192, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 368523776, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.6.ffn_gate.weight", "offset": 371792384, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 375060992, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 378329600, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 378333184, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 378333696, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 378419712, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 379021824, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 379025408, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.6.attn_v.bias", "offset": 379627520, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 379628032, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 379714048, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 379717632, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.7.ffn_gate.weight", "offset": 382986240, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 386254848, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 389523456, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 389527040, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 389527552, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.7.attn_output.weight", "offset": 389613568, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 390215680, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 390219264, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.7.attn_v.bias", "offset": 390821376, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 390821888, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 390907904, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 390911488, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.8.ffn_gate.weight", "offset": 394180096, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 397448704, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 400717312, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 400720896, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 400721408, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.8.attn_output.weight", "offset": 400807424, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 401409536, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 401413120, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.8.attn_v.bias", "offset": 402015232, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 402015744, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 402101760, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 402105344, "shape": [ 4864, 896 ], "size": 3584, "type": 7 }, { "name": "blk.9.ffn_gate.weight", "offset": 405373952, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 408642560, "shape": [ 896, 4864 ], "size": 3268608, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 411911168, "shape": [ 896 ], "size": 3268608, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 411914752, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 411915264, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.9.attn_output.weight", "offset": 412001280, "shape": [ 896, 896 ], "size": 86016, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 412603392, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 412606976, "shape": [ 896, 896 ], "size": 3584, "type": 7 }, { "name": "blk.9.attn_v.bias", "offset": 413209088, "shape": [ 128 ], "size": 602112, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 413209600, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "output_norm.weight", "offset": 413295616, "shape": [ 896 ], "size": 86016, "type": 0 } ], "version": 3 }