qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 11, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 146519296, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.0.ffn_up.weight", "offset": 148970752, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.0.ffn_norm.weight", "offset": 151422208, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 151425792, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 151426304, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.0.attn_output.weight", "offset": 151490816, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.0.attn_q.bias", "offset": 151942400, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 151945984, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.0.attn_v.bias", "offset": 152397568, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 152398080, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.1.attn_norm.weight", "offset": 152462592, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 152466176, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 154338816, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.1.ffn_up.weight", "offset": 156790272, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.1.ffn_norm.weight", "offset": 159241728, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 159245312, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 159245824, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.1.attn_output.weight", "offset": 159310336, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.1.attn_q.bias", "offset": 159761920, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 159765504, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.1.attn_v.bias", "offset": 160217088, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 160217600, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.10.attn_norm.weight", "offset": 160282112, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 160285696, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 162158336, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.10.ffn_up.weight", "offset": 164609792, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.10.ffn_norm.weight", "offset": 167061248, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 167064832, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 167065344, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.10.attn_output.weight", "offset": 167129856, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.10.attn_q.bias", "offset": 167581440, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 167585024, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.10.attn_v.bias", "offset": 168036608, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 168037120, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.11.attn_norm.weight", "offset": 168101632, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 168105216, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 169977856, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.11.ffn_up.weight", "offset": 172429312, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.11.ffn_norm.weight", "offset": 174880768, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 174884352, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 174884864, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.11.attn_output.weight", "offset": 174949376, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.11.attn_q.bias", "offset": 175400960, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 175404544, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.11.attn_v.bias", "offset": 175856128, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 175856640, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.12.attn_norm.weight", "offset": 175921152, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 175924736, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 177797376, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.12.ffn_up.weight", "offset": 180248832, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.12.ffn_norm.weight", "offset": 182700288, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 182703872, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 182704384, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.12.attn_output.weight", "offset": 182768896, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.12.attn_q.bias", "offset": 183220480, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 183224064, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.12.attn_v.bias", "offset": 183675648, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 183676160, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.13.attn_norm.weight", "offset": 183740672, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 183744256, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 185616896, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.13.ffn_up.weight", "offset": 188068352, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.13.ffn_norm.weight", "offset": 190519808, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 190523392, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 190523904, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.13.attn_output.weight", "offset": 190588416, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.13.attn_q.bias", "offset": 191040000, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 191043584, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.13.attn_v.bias", "offset": 191495168, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 191495680, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.14.attn_norm.weight", "offset": 191560192, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 191563776, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 193436416, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.14.ffn_up.weight", "offset": 195887872, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.14.ffn_norm.weight", "offset": 198339328, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 198342912, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 198343424, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.14.attn_output.weight", "offset": 198407936, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.14.attn_q.bias", "offset": 198859520, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 198863104, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.14.attn_v.bias", "offset": 199314688, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 199315200, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.15.attn_norm.weight", "offset": 199379712, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 199383296, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 201255936, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.15.ffn_up.weight", "offset": 203707392, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.15.ffn_norm.weight", "offset": 206158848, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 206162432, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 206162944, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.15.attn_output.weight", "offset": 206227456, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.15.attn_q.bias", "offset": 206679040, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 206682624, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.15.attn_v.bias", "offset": 207134208, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 207134720, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.16.attn_norm.weight", "offset": 207199232, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 207202816, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 209075456, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.16.ffn_up.weight", "offset": 211526912, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.16.ffn_norm.weight", "offset": 213978368, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 213981952, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 213982464, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.16.attn_output.weight", "offset": 214046976, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.16.attn_q.bias", "offset": 214498560, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 214502144, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.16.attn_v.bias", "offset": 214953728, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 214954240, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.17.attn_norm.weight", "offset": 215018752, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 215022336, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 216894976, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.17.ffn_up.weight", "offset": 219346432, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.17.ffn_norm.weight", "offset": 221797888, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 221801472, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 221801984, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.17.attn_output.weight", "offset": 221866496, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.17.attn_q.bias", "offset": 222318080, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 222321664, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.17.attn_v.bias", "offset": 222773248, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 222773760, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.18.attn_norm.weight", "offset": 222838272, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 222841856, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 224714496, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.18.ffn_up.weight", "offset": 227165952, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.18.ffn_norm.weight", "offset": 229617408, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 229620992, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 229621504, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.18.attn_output.weight", "offset": 229686016, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.18.attn_q.bias", "offset": 230137600, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 230141184, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.18.attn_v.bias", "offset": 230592768, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 230593280, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.19.attn_norm.weight", "offset": 230657792, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 230661376, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 232534016, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.19.ffn_up.weight", "offset": 234985472, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.19.ffn_norm.weight", "offset": 237436928, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 237440512, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 237441024, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.19.attn_output.weight", "offset": 237505536, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.19.attn_q.bias", "offset": 237957120, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 237960704, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.19.attn_v.bias", "offset": 238412288, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 238412800, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.2.attn_norm.weight", "offset": 238477312, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 238480896, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 240353536, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.2.ffn_up.weight", "offset": 242804992, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.2.ffn_norm.weight", "offset": 245256448, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 245260032, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 245260544, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.2.attn_output.weight", "offset": 245325056, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.2.attn_q.bias", "offset": 245776640, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 245780224, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.2.attn_v.bias", "offset": 246231808, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 246232320, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.20.attn_norm.weight", "offset": 246296832, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 246300416, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 248173056, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.20.ffn_up.weight", "offset": 250624512, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.20.ffn_norm.weight", "offset": 253075968, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 253079552, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 253080064, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.20.attn_output.weight", "offset": 253144576, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.20.attn_q.bias", "offset": 253596160, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 253599744, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.20.attn_v.bias", "offset": 254051328, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 254051840, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.21.attn_norm.weight", "offset": 254116352, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 254119936, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 255992576, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.21.ffn_up.weight", "offset": 258444032, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.21.ffn_norm.weight", "offset": 260895488, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 260899072, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 260899584, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.21.attn_output.weight", "offset": 260964096, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.21.attn_q.bias", "offset": 261415680, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 261419264, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.21.attn_v.bias", "offset": 261870848, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 261871360, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.22.attn_norm.weight", "offset": 261935872, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 261939456, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 263812096, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.22.ffn_up.weight", "offset": 266263552, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.22.ffn_norm.weight", "offset": 268715008, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 268718592, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 268719104, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.22.attn_output.weight", "offset": 268783616, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.22.attn_q.bias", "offset": 269235200, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 269238784, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.22.attn_v.bias", "offset": 269690368, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 269690880, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.23.attn_norm.weight", "offset": 269755392, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 269758976, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 271631616, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.23.ffn_up.weight", "offset": 274083072, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.23.ffn_norm.weight", "offset": 276534528, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 276538112, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 276538624, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.23.attn_output.weight", "offset": 276603136, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.23.attn_q.bias", "offset": 277054720, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 277058304, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.23.attn_v.bias", "offset": 277509888, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 277510400, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.3.attn_norm.weight", "offset": 277574912, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 277578496, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 279451136, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.3.ffn_up.weight", "offset": 281902592, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.3.ffn_norm.weight", "offset": 284354048, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 284357632, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 284358144, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.3.attn_output.weight", "offset": 284422656, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.3.attn_q.bias", "offset": 284874240, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 284877824, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.3.attn_v.bias", "offset": 285329408, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 285329920, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.4.attn_norm.weight", "offset": 285394432, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 285398016, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 287270656, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.4.ffn_up.weight", "offset": 289722112, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.4.ffn_norm.weight", "offset": 292173568, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 292177152, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 292177664, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.4.attn_output.weight", "offset": 292242176, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.4.attn_q.bias", "offset": 292693760, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 292697344, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.4.attn_v.bias", "offset": 293148928, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 293149440, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.5.attn_norm.weight", "offset": 293213952, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 293217536, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 295090176, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.5.ffn_up.weight", "offset": 297541632, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.5.ffn_norm.weight", "offset": 299993088, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 299996672, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 299997184, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.5.attn_output.weight", "offset": 300061696, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.5.attn_q.bias", "offset": 300513280, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 300516864, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.5.attn_v.bias", "offset": 300968448, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 300968960, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.6.attn_norm.weight", "offset": 301033472, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 301037056, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 302909696, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.6.ffn_up.weight", "offset": 305361152, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.6.ffn_norm.weight", "offset": 307812608, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 307816192, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 307816704, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.6.attn_output.weight", "offset": 307881216, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.6.attn_q.bias", "offset": 308332800, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 308336384, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.6.attn_v.bias", "offset": 308787968, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 308788480, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.7.attn_norm.weight", "offset": 308852992, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 308856576, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 310729216, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.7.ffn_up.weight", "offset": 313180672, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.7.ffn_norm.weight", "offset": 315632128, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 315635712, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 315636224, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.7.attn_output.weight", "offset": 315700736, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.7.attn_q.bias", "offset": 316152320, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 316155904, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.7.attn_v.bias", "offset": 316607488, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 316608000, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.8.attn_norm.weight", "offset": 316672512, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 316676096, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 318548736, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.8.ffn_up.weight", "offset": 321000192, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.8.ffn_norm.weight", "offset": 323451648, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 323455232, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 323455744, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.8.attn_output.weight", "offset": 323520256, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.8.attn_q.bias", "offset": 323971840, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 323975424, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.8.attn_v.bias", "offset": 324427008, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 324427520, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.9.attn_norm.weight", "offset": 324492032, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 324495616, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 326368256, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.9.ffn_up.weight", "offset": 328819712, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.9.ffn_norm.weight", "offset": 331271168, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 331274752, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 331275264, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.9.attn_output.weight", "offset": 331339776, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.9.attn_q.bias", "offset": 331791360, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 331794944, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.9.attn_v.bias", "offset": 332246528, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 332247040, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "output_norm.weight", "offset": 332311552, "shape": [ 896 ], "size": 64512, "type": 0 } ], "version": 3 }