qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 12, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 147642880, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.0.ffn_up.weight", "offset": 150094336, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.0.ffn_norm.weight", "offset": 152545792, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 152549376, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 152549888, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.0.attn_output.weight", "offset": 152614400, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.0.attn_q.bias", "offset": 153166336, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 153169920, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.0.attn_v.bias", "offset": 153621504, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 153622016, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 153708032, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 153711616, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.1.ffn_gate.weight", "offset": 156163072, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.1.ffn_up.weight", "offset": 158614528, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.1.ffn_norm.weight", "offset": 161065984, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 161069568, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 161070080, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.1.attn_output.weight", "offset": 161134592, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.1.attn_q.bias", "offset": 161686528, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 161690112, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.1.attn_v.bias", "offset": 162141696, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 162142208, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 162228224, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 162231808, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 164683264, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.10.ffn_up.weight", "offset": 167134720, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.10.ffn_norm.weight", "offset": 169586176, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 169589760, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 169590272, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.10.attn_output.weight", "offset": 169654784, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.10.attn_q.bias", "offset": 170206720, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 170210304, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.10.attn_v.bias", "offset": 170661888, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 170662400, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 170741248, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 170744832, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 173196288, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.11.ffn_up.weight", "offset": 175647744, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.11.ffn_norm.weight", "offset": 178099200, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 178102784, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 178103296, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.11.attn_output.weight", "offset": 178167808, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.11.attn_q.bias", "offset": 178719744, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 178723328, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.11.attn_v.bias", "offset": 179174912, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 179175424, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 179254272, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 179257856, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 181709312, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.12.ffn_up.weight", "offset": 184160768, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.12.ffn_norm.weight", "offset": 186612224, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 186615808, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 186616320, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.12.attn_output.weight", "offset": 186680832, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.12.attn_q.bias", "offset": 187232768, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 187236352, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.12.attn_v.bias", "offset": 187687936, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 187688448, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 187767296, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 187770880, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 190222336, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.13.ffn_up.weight", "offset": 192673792, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.13.ffn_norm.weight", "offset": 195125248, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 195128832, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 195129344, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.13.attn_output.weight", "offset": 195193856, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.13.attn_q.bias", "offset": 195745792, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 195749376, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.13.attn_v.bias", "offset": 196200960, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 196201472, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 196280320, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 196283904, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 198735360, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.14.ffn_up.weight", "offset": 201186816, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.14.ffn_norm.weight", "offset": 203638272, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 203641856, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 203642368, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.14.attn_output.weight", "offset": 203706880, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.14.attn_q.bias", "offset": 204258816, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 204262400, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.14.attn_v.bias", "offset": 204713984, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 204714496, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 204793344, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 204796928, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 207248384, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.15.ffn_up.weight", "offset": 209699840, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.15.ffn_norm.weight", "offset": 212151296, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 212154880, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 212155392, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.15.attn_output.weight", "offset": 212219904, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.15.attn_q.bias", "offset": 212771840, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 212775424, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.15.attn_v.bias", "offset": 213227008, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 213227520, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 213306368, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 213309952, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 215761408, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.16.ffn_up.weight", "offset": 218212864, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.16.ffn_norm.weight", "offset": 220664320, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 220667904, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 220668416, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.16.attn_output.weight", "offset": 220732928, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.16.attn_q.bias", "offset": 221284864, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 221288448, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.16.attn_v.bias", "offset": 221740032, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 221740544, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 221819392, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 221822976, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 224274432, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.17.ffn_up.weight", "offset": 226725888, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.17.ffn_norm.weight", "offset": 229177344, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 229180928, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 229181440, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.17.attn_output.weight", "offset": 229245952, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.17.attn_q.bias", "offset": 229797888, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 229801472, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.17.attn_v.bias", "offset": 230253056, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 230253568, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 230332416, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 230336000, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 232787456, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.18.ffn_up.weight", "offset": 235238912, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.18.ffn_norm.weight", "offset": 237690368, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 237693952, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 237694464, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.18.attn_output.weight", "offset": 237758976, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.18.attn_q.bias", "offset": 238310912, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 238314496, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.18.attn_v.bias", "offset": 238766080, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 238766592, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 238845440, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 238849024, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 241300480, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.19.ffn_up.weight", "offset": 243751936, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.19.ffn_norm.weight", "offset": 246203392, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 246206976, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 246207488, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.19.attn_output.weight", "offset": 246272000, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.19.attn_q.bias", "offset": 246823936, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 246827520, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.19.attn_v.bias", "offset": 247279104, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 247279616, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 247358464, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 247362048, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 249813504, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.2.ffn_up.weight", "offset": 252264960, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.2.ffn_norm.weight", "offset": 254716416, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 254720000, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 254720512, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.2.attn_output.weight", "offset": 254785024, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.2.attn_q.bias", "offset": 255336960, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 255340544, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.2.attn_v.bias", "offset": 255792128, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 255792640, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 255871488, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 255875072, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 258326528, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.20.ffn_up.weight", "offset": 260777984, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.20.ffn_norm.weight", "offset": 263229440, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 263233024, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 263233536, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.20.attn_output.weight", "offset": 263298048, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.20.attn_q.bias", "offset": 263849984, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 263853568, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.20.attn_v.bias", "offset": 264305152, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 264305664, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 264384512, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 264388096, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 266839552, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.21.ffn_up.weight", "offset": 269291008, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.21.ffn_norm.weight", "offset": 271742464, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 271746048, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 271746560, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.21.attn_output.weight", "offset": 271811072, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.21.attn_q.bias", "offset": 272363008, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 272366592, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.21.attn_v.bias", "offset": 272818176, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 272818688, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 272897536, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 272901120, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 275352576, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.22.ffn_up.weight", "offset": 277804032, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.22.ffn_norm.weight", "offset": 280255488, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 280259072, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 280259584, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.22.attn_output.weight", "offset": 280324096, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.22.attn_q.bias", "offset": 280876032, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 280879616, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.22.attn_v.bias", "offset": 281331200, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 281331712, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 281410560, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 281414144, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 283865600, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.23.ffn_up.weight", "offset": 286317056, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.23.ffn_norm.weight", "offset": 288768512, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 288772096, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 288772608, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.23.attn_output.weight", "offset": 288837120, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.23.attn_q.bias", "offset": 289389056, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 289392640, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.23.attn_v.bias", "offset": 289844224, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 289844736, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 289923584, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 289927168, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 292378624, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.3.ffn_up.weight", "offset": 294830080, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.3.ffn_norm.weight", "offset": 297281536, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 297285120, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 297285632, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.3.attn_output.weight", "offset": 297350144, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.3.attn_q.bias", "offset": 297902080, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 297905664, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.3.attn_v.bias", "offset": 298357248, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 298357760, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 298436608, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 298440192, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 300891648, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.4.ffn_up.weight", "offset": 303343104, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.4.ffn_norm.weight", "offset": 305794560, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 305798144, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 305798656, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.4.attn_output.weight", "offset": 305863168, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.4.attn_q.bias", "offset": 306415104, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 306418688, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.4.attn_v.bias", "offset": 306870272, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 306870784, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 306949632, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 306953216, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 309404672, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.5.ffn_up.weight", "offset": 311856128, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.5.ffn_norm.weight", "offset": 314307584, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 314311168, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 314311680, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.5.attn_output.weight", "offset": 314376192, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.5.attn_q.bias", "offset": 314928128, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 314931712, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.5.attn_v.bias", "offset": 315383296, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 315383808, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 315462656, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 315466240, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 317917696, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.6.ffn_up.weight", "offset": 320369152, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.6.ffn_norm.weight", "offset": 322820608, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 322824192, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 322824704, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.6.attn_output.weight", "offset": 322889216, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.6.attn_q.bias", "offset": 323441152, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 323444736, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.6.attn_v.bias", "offset": 323896320, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 323896832, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.7.attn_norm.weight", "offset": 323975680, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 323979264, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 326430720, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.7.ffn_up.weight", "offset": 328882176, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.7.ffn_norm.weight", "offset": 331333632, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 331337216, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 331337728, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.7.attn_output.weight", "offset": 331402240, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.7.attn_q.bias", "offset": 331954176, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 331957760, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.7.attn_v.bias", "offset": 332409344, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 332409856, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 332488704, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 332492288, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 334943744, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.8.ffn_up.weight", "offset": 337395200, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.8.ffn_norm.weight", "offset": 339846656, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 339850240, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 339850752, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.8.attn_output.weight", "offset": 339915264, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.8.attn_q.bias", "offset": 340467200, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 340470784, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.8.attn_v.bias", "offset": 340922368, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 340922880, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 341001728, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 341005312, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 343456768, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.9.ffn_up.weight", "offset": 345908224, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.9.ffn_norm.weight", "offset": 348359680, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 348363264, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 348363776, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.9.attn_output.weight", "offset": 348428288, "shape": [ 896, 896 ], "size": 64512, "type": 6 }, { "name": "blk.9.attn_q.bias", "offset": 348980224, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 348983808, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.9.attn_v.bias", "offset": 349435392, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 349435904, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "output_norm.weight", "offset": 349514752, "shape": [ 896 ], "size": 78848, "type": 0 } ], "version": 3 }