qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 10, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.0.ffn_gate.weight", "offset": 146519296, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.0.ffn_up.weight", "offset": 148970752, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.0.ffn_norm.weight", "offset": 151422208, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 151425792, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 151426304, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.0.attn_output.weight", "offset": 151490816, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.0.attn_q.bias", "offset": 151942400, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 151945984, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.0.attn_v.bias", "offset": 152397568, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 152398080, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 152476928, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 152480512, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.1.ffn_gate.weight", "offset": 154353152, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.1.ffn_up.weight", "offset": 156804608, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.1.ffn_norm.weight", "offset": 159256064, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 159259648, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 159260160, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.1.attn_output.weight", "offset": 159324672, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.1.attn_q.bias", "offset": 159776256, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 159779840, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.1.attn_v.bias", "offset": 160231424, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 160231936, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 160310784, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 160314368, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.10.ffn_gate.weight", "offset": 162187008, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.10.ffn_up.weight", "offset": 164638464, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.10.ffn_norm.weight", "offset": 167089920, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 167093504, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 167094016, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.10.attn_output.weight", "offset": 167158528, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.10.attn_q.bias", "offset": 167610112, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 167613696, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.10.attn_v.bias", "offset": 168065280, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 168065792, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 168144640, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 168148224, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.11.ffn_gate.weight", "offset": 170020864, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.11.ffn_up.weight", "offset": 172472320, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.11.ffn_norm.weight", "offset": 174923776, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 174927360, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 174927872, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.11.attn_output.weight", "offset": 174992384, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.11.attn_q.bias", "offset": 175443968, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 175447552, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.11.attn_v.bias", "offset": 175899136, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 175899648, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 175978496, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 175982080, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.12.ffn_gate.weight", "offset": 177854720, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.12.ffn_up.weight", "offset": 180306176, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.12.ffn_norm.weight", "offset": 182757632, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 182761216, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 182761728, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.12.attn_output.weight", "offset": 182826240, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.12.attn_q.bias", "offset": 183277824, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 183281408, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.12.attn_v.bias", "offset": 183732992, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 183733504, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 183812352, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 183815936, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.13.ffn_gate.weight", "offset": 185688576, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.13.ffn_up.weight", "offset": 188140032, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.13.ffn_norm.weight", "offset": 190591488, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 190595072, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 190595584, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.13.attn_output.weight", "offset": 190660096, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.13.attn_q.bias", "offset": 191111680, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 191115264, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.13.attn_v.bias", "offset": 191566848, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 191567360, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 191646208, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 191649792, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.14.ffn_gate.weight", "offset": 193522432, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.14.ffn_up.weight", "offset": 195973888, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.14.ffn_norm.weight", "offset": 198425344, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 198428928, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 198429440, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.14.attn_output.weight", "offset": 198493952, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.14.attn_q.bias", "offset": 198945536, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 198949120, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.14.attn_v.bias", "offset": 199400704, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 199401216, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 199480064, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 199483648, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.15.ffn_gate.weight", "offset": 201356288, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.15.ffn_up.weight", "offset": 203807744, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.15.ffn_norm.weight", "offset": 206259200, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 206262784, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 206263296, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.15.attn_output.weight", "offset": 206327808, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.15.attn_q.bias", "offset": 206779392, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 206782976, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.15.attn_v.bias", "offset": 207234560, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 207235072, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 207313920, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 207317504, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.16.ffn_gate.weight", "offset": 209190144, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.16.ffn_up.weight", "offset": 211641600, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.16.ffn_norm.weight", "offset": 214093056, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 214096640, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 214097152, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.16.attn_output.weight", "offset": 214161664, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.16.attn_q.bias", "offset": 214613248, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 214616832, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.16.attn_v.bias", "offset": 215068416, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 215068928, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 215147776, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 215151360, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.17.ffn_gate.weight", "offset": 217024000, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.17.ffn_up.weight", "offset": 219475456, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.17.ffn_norm.weight", "offset": 221926912, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 221930496, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 221931008, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.17.attn_output.weight", "offset": 221995520, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.17.attn_q.bias", "offset": 222447104, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 222450688, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.17.attn_v.bias", "offset": 222902272, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 222902784, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 222981632, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 222985216, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.18.ffn_gate.weight", "offset": 224857856, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.18.ffn_up.weight", "offset": 227309312, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.18.ffn_norm.weight", "offset": 229760768, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 229764352, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 229764864, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.18.attn_output.weight", "offset": 229829376, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.18.attn_q.bias", "offset": 230280960, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 230284544, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.18.attn_v.bias", "offset": 230736128, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 230736640, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 230815488, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 230819072, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.19.ffn_gate.weight", "offset": 232691712, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.19.ffn_up.weight", "offset": 235143168, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.19.ffn_norm.weight", "offset": 237594624, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 237598208, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 237598720, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.19.attn_output.weight", "offset": 237663232, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.19.attn_q.bias", "offset": 238114816, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 238118400, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.19.attn_v.bias", "offset": 238569984, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 238570496, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 238649344, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 238652928, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.2.ffn_gate.weight", "offset": 240525568, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.2.ffn_up.weight", "offset": 242977024, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.2.ffn_norm.weight", "offset": 245428480, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 245432064, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 245432576, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.2.attn_output.weight", "offset": 245497088, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.2.attn_q.bias", "offset": 245948672, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 245952256, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.2.attn_v.bias", "offset": 246403840, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 246404352, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 246483200, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 246486784, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.20.ffn_gate.weight", "offset": 248359424, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.20.ffn_up.weight", "offset": 250810880, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.20.ffn_norm.weight", "offset": 253262336, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 253265920, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 253266432, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.20.attn_output.weight", "offset": 253330944, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.20.attn_q.bias", "offset": 253782528, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 253786112, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.20.attn_v.bias", "offset": 254237696, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 254238208, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 254317056, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 254320640, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.21.ffn_gate.weight", "offset": 256193280, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.21.ffn_up.weight", "offset": 258644736, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.21.ffn_norm.weight", "offset": 261096192, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 261099776, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 261100288, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.21.attn_output.weight", "offset": 261164800, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.21.attn_q.bias", "offset": 261616384, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 261619968, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.21.attn_v.bias", "offset": 262071552, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 262072064, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 262150912, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 262154496, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.22.ffn_gate.weight", "offset": 264027136, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.22.ffn_up.weight", "offset": 266478592, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.22.ffn_norm.weight", "offset": 268930048, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 268933632, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 268934144, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.22.attn_output.weight", "offset": 268998656, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.22.attn_q.bias", "offset": 269450240, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 269453824, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.22.attn_v.bias", "offset": 269905408, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 269905920, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 269984768, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 269988352, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.23.ffn_gate.weight", "offset": 271860992, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.23.ffn_up.weight", "offset": 274312448, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.23.ffn_norm.weight", "offset": 276763904, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 276767488, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 276768000, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.23.attn_output.weight", "offset": 276832512, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.23.attn_q.bias", "offset": 277284096, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 277287680, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.23.attn_v.bias", "offset": 277739264, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 277739776, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 277818624, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 277822208, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.3.ffn_gate.weight", "offset": 279694848, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.3.ffn_up.weight", "offset": 282146304, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.3.ffn_norm.weight", "offset": 284597760, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 284601344, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 284601856, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.3.attn_output.weight", "offset": 284666368, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.3.attn_q.bias", "offset": 285117952, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 285121536, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.3.attn_v.bias", "offset": 285573120, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 285573632, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 285652480, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 285656064, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.4.ffn_gate.weight", "offset": 287528704, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.4.ffn_up.weight", "offset": 289980160, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.4.ffn_norm.weight", "offset": 292431616, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 292435200, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 292435712, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.4.attn_output.weight", "offset": 292500224, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.4.attn_q.bias", "offset": 292951808, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 292955392, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.4.attn_v.bias", "offset": 293406976, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 293407488, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 293486336, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 293489920, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.5.ffn_gate.weight", "offset": 295362560, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.5.ffn_up.weight", "offset": 297814016, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.5.ffn_norm.weight", "offset": 300265472, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 300269056, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 300269568, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.5.attn_output.weight", "offset": 300334080, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.5.attn_q.bias", "offset": 300785664, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 300789248, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.5.attn_v.bias", "offset": 301240832, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 301241344, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 301320192, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 301323776, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.6.ffn_gate.weight", "offset": 303196416, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.6.ffn_up.weight", "offset": 305647872, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.6.ffn_norm.weight", "offset": 308099328, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 308102912, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 308103424, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.6.attn_output.weight", "offset": 308167936, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.6.attn_q.bias", "offset": 308619520, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 308623104, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.6.attn_v.bias", "offset": 309074688, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 309075200, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.7.attn_norm.weight", "offset": 309154048, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 309157632, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.7.ffn_gate.weight", "offset": 311030272, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.7.ffn_up.weight", "offset": 313481728, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.7.ffn_norm.weight", "offset": 315933184, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 315936768, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 315937280, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.7.attn_output.weight", "offset": 316001792, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.7.attn_q.bias", "offset": 316453376, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 316456960, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.7.attn_v.bias", "offset": 316908544, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 316909056, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 316987904, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 316991488, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.8.ffn_gate.weight", "offset": 318864128, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.8.ffn_up.weight", "offset": 321315584, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.8.ffn_norm.weight", "offset": 323767040, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 323770624, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 323771136, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.8.attn_output.weight", "offset": 323835648, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.8.attn_q.bias", "offset": 324287232, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 324290816, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.8.attn_v.bias", "offset": 324742400, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 324742912, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 324821760, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 324825344, "shape": [ 4864, 896 ], "size": 3584, "type": 11 }, { "name": "blk.9.ffn_gate.weight", "offset": 326697984, "shape": [ 896, 4864 ], "size": 1872640, "type": 20 }, { "name": "blk.9.ffn_up.weight", "offset": 329149440, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.9.ffn_norm.weight", "offset": 331600896, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 331604480, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 331604992, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.9.attn_output.weight", "offset": 331669504, "shape": [ 896, 896 ], "size": 64512, "type": 20 }, { "name": "blk.9.attn_q.bias", "offset": 332121088, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 332124672, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.9.attn_v.bias", "offset": 332576256, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 332576768, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "output_norm.weight", "offset": 332655616, "shape": [ 896 ], "size": 78848, "type": 0 } ], "version": 3 }