qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 15, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 148221696, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 151217920, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 154214144, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 154217728, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 154218240, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 154297088, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.0.attn_q.bias", "offset": 154849024, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 154852608, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.0.attn_v.bias", "offset": 155404544, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 155405056, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 155526912, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 155530496, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 159105536, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 162101760, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 165097984, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 165101568, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 165102080, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 165180928, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.1.attn_q.bias", "offset": 165732864, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 165736448, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.1.attn_v.bias", "offset": 166288384, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 166288896, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 166410752, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 166414336, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 169989376, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 172985600, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 175981824, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 175985408, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 175985920, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 176064768, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.10.attn_q.bias", "offset": 176616704, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 176620288, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.10.attn_v.bias", "offset": 177172224, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 177172736, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 177294592, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 177298176, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "\u0000\u0000\u0000\ufffd\u001b\u000b\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.", "offset": 179749632, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 182745856, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 185742080, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 185745664, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 185746176, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 185825024, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.11.attn_q.bias", "offset": 186376960, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 186380544, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.11.attn_v.bias", "offset": 186932480, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 186932992, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 187011840, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 187015424, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 189466880, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 192463104, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 195459328, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 195462912, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 195463424, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 195542272, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.12.attn_q.bias", "offset": 196094208, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 196097792, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.12.attn_v.bias", "offset": 196649728, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 196650240, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 196729088, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 196732672, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 200307712, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 203303936, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 206300160, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 206303744, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 206304256, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 206383104, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.13.attn_q.bias", "offset": 206935040, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 206938624, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.13.attn_v.bias", "offset": 207490560, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 207491072, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 207612928, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 207616512, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 210067968, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 213064192, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 216060416, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 216064000, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 216064512, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 216143360, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.14.attn_q.bias", "offset": 216695296, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 216698880, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.14.attn_v.bias", "offset": 217250816, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 217251328, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 217330176, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 217333760, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 219785216, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 222781440, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 225777664, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 225781248, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 225781760, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 225860608, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.15.attn_q.bias", "offset": 226412544, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 226416128, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.15.attn_v.bias", "offset": 226968064, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 226968576, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 227047424, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 227051008, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 230626048, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 233622272, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 236618496, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 236622080, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 236622592, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 236701440, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.16.attn_q.bias", "offset": 237253376, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 237256960, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.16.attn_v.bias", "offset": 237808896, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 237809408, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 237931264, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 237934848, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 240386304, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 243382528, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 246378752, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 246382336, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 246382848, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 246461696, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.17.attn_q.bias", "offset": 247013632, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 247017216, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.17.attn_v.bias", "offset": 247569152, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 247569664, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 247648512, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 247652096, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 250103552, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 253099776, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 256096000, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 256099584, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 256100096, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 256178944, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.18.attn_q.bias", "offset": 256730880, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 256734464, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.18.attn_v.bias", "offset": 257286400, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 257286912, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 257365760, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 257369344, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 260944384, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 263940608, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 266936832, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 266940416, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 266940928, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 267019776, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.19.attn_q.bias", "offset": 267571712, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 267575296, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.19.attn_v.bias", "offset": 268127232, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 268127744, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 268249600, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 268253184, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 270704640, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 273700864, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 276697088, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 276700672, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 276701184, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 276780032, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.2.attn_q.bias", "offset": 277331968, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 277335552, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.2.attn_v.bias", "offset": 277887488, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 277888000, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 277966848, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 277970432, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 280421888, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 283418112, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 286414336, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 286417920, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 286418432, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 286497280, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.20.attn_q.bias", "offset": 287049216, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 287052800, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.20.attn_v.bias", "offset": 287604736, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 287605248, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 287684096, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 287687680, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 291262720, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 294258944, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 297255168, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 297258752, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 297259264, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 297338112, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.21.attn_q.bias", "offset": 297890048, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 297893632, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.21.attn_v.bias", "offset": 298445568, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 298446080, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 298567936, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 298571520, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 301022976, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 304019200, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 307015424, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 307019008, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 307019520, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 307098368, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.22.attn_q.bias", "offset": 307650304, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 307653888, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.22.attn_v.bias", "offset": 308205824, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 308206336, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 308285184, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 308288768, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 310740224, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 313736448, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 316732672, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 316736256, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 316736768, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 316815616, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.23.attn_q.bias", "offset": 317367552, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 317371136, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.23.attn_v.bias", "offset": 317923072, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 317923584, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 318002432, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 318006016, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 321581056, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 324577280, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 327573504, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 327577088, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 327577600, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 327656448, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.3.attn_q.bias", "offset": 328208384, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 328211968, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.3.attn_v.bias", "offset": 328763904, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 328764416, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 328886272, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 328889856, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 331341312, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 334337536, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 337333760, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 337337344, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 337337856, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 337416704, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.4.attn_q.bias", "offset": 337968640, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 337972224, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.4.attn_v.bias", "offset": 338524160, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 338524672, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 338603520, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 338607104, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 341058560, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 344054784, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 347051008, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 347054592, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 347055104, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 347133952, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.5.attn_q.bias", "offset": 347685888, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 347689472, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.5.attn_v.bias", "offset": 348241408, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 348241920, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 348320768, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 348324352, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 351899392, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 354895616, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 357891840, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 357895424, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 357895936, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 357974784, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.6.attn_q.bias", "offset": 358526720, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 358530304, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.6.attn_v.bias", "offset": 359082240, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 359082752, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 359204608, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 359208192, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 362783232, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 365779456, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 368775680, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 368779264, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 368779776, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 368858624, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.7.attn_q.bias", "offset": 369410560, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 369414144, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.7.attn_v.bias", "offset": 369966080, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 369966592, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 370088448, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 370092032, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 373667072, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 376663296, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 379659520, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 379663104, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 379663616, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 379742464, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.8.attn_q.bias", "offset": 380294400, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 380297984, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.8.attn_v.bias", "offset": 380849920, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 380850432, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 380972288, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 380975872, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 384550912, "shape": [ 896, 4864 ], "size": 3575040, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 387547136, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 390543360, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 390546944, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 390547456, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 390626304, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.9.attn_q.bias", "offset": 391178240, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 391181824, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.9.attn_v.bias", "offset": 391733760, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 391734272, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "output_norm.weight", "offset": 391856128, "shape": [ 896 ], "size": 121856, "type": 0 } ], "version": 3 }