qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 14, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 147642880, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 150639104, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 153635328, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 153638912, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 153639424, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.0.attn_output.weight", "offset": 153718272, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.0.attn_q.bias", "offset": 154270208, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 154273792, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.0.attn_v.bias", "offset": 154825728, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 154826240, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 154912256, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 154915840, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 157912064, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 160908288, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 163904512, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 163908096, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 163908608, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.1.attn_output.weight", "offset": 163987456, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.1.attn_q.bias", "offset": 164539392, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 164542976, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.1.attn_v.bias", "offset": 165094912, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 165095424, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 165181440, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 165185024, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 168181248, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 171177472, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 174173696, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 174177280, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 174177792, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.10.attn_output.weight", "offset": 174256640, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.10.attn_q.bias", "offset": 174808576, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 174812160, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.10.attn_v.bias", "offset": 175364096, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 175364608, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 175450624, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 175454208, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 177905664, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 180901888, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 183898112, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 183901696, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 183902208, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.11.attn_output.weight", "offset": 183981056, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.11.attn_q.bias", "offset": 184532992, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 184536576, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.11.attn_v.bias", "offset": 185088512, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 185089024, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 185175040, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 185178624, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 187630080, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 190626304, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 193622528, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 193626112, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 193626624, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.12.attn_output.weight", "offset": 193705472, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.12.attn_q.bias", "offset": 194257408, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 194260992, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.12.attn_v.bias", "offset": 194812928, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 194813440, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 194892288, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 194895872, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 197347328, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 200343552, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 203339776, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 203343360, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 203343872, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.13.attn_output.weight", "offset": 203422720, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.13.attn_q.bias", "offset": 203974656, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 203978240, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.13.attn_v.bias", "offset": 204530176, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 204530688, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 204609536, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 204613120, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 207064576, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 210060800, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 213057024, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 213060608, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 213061120, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.14.attn_output.weight", "offset": 213139968, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.14.attn_q.bias", "offset": 213691904, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 213695488, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.14.attn_v.bias", "offset": 214247424, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 214247936, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 214326784, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 214330368, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 216781824, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 219778048, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 222774272, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 222777856, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 222778368, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.15.attn_output.weight", "offset": 222857216, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.15.attn_q.bias", "offset": 223409152, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 223412736, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.15.attn_v.bias", "offset": 223964672, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 223965184, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 224044032, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 224047616, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 226499072, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 229495296, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 232491520, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 232495104, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 232495616, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.16.attn_output.weight", "offset": 232574464, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.16.attn_q.bias", "offset": 233126400, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 233129984, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.16.attn_v.bias", "offset": 233681920, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 233682432, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 233761280, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 233764864, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 236216320, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 239212544, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 242208768, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 242212352, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 242212864, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.17.attn_output.weight", "offset": 242291712, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.17.attn_q.bias", "offset": 242843648, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 242847232, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.17.attn_v.bias", "offset": 243399168, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 243399680, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 243478528, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 243482112, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 245933568, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 248929792, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 251926016, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 251929600, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 251930112, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.18.attn_output.weight", "offset": 252008960, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.18.attn_q.bias", "offset": 252560896, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 252564480, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.18.attn_v.bias", "offset": 253116416, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 253116928, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 253195776, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 253199360, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 255650816, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 258647040, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 261643264, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 261646848, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 261647360, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.19.attn_output.weight", "offset": 261726208, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.19.attn_q.bias", "offset": 262278144, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 262281728, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.19.attn_v.bias", "offset": 262833664, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 262834176, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 262913024, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 262916608, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 265368064, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 268364288, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 271360512, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 271364096, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 271364608, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.2.attn_output.weight", "offset": 271443456, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.2.attn_q.bias", "offset": 271995392, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 271998976, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.2.attn_v.bias", "offset": 272550912, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 272551424, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 272630272, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 272633856, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 275085312, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 278081536, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 281077760, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 281081344, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 281081856, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.20.attn_output.weight", "offset": 281160704, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.20.attn_q.bias", "offset": 281712640, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 281716224, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.20.attn_v.bias", "offset": 282268160, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 282268672, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 282347520, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 282351104, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 284802560, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 287798784, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 290795008, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 290798592, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 290799104, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 290877952, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.21.attn_q.bias", "offset": 291429888, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 291433472, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.21.attn_v.bias", "offset": 291985408, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 291985920, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.22.attn_norm.weight", "offset": 292064768, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 292068352, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 294519808, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 297516032, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 300512256, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 300515840, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 300516352, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.22.attn_output.weight", "offset": 300595200, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.22.attn_q.bias", "offset": 301147136, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 301150720, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.22.attn_v.bias", "offset": 301702656, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 301703168, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 301782016, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 301785600, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 304237056, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 307233280, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 310229504, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 310233088, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 310233600, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.23.attn_output.weight", "offset": 310312448, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.23.attn_q.bias", "offset": 310864384, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 310867968, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.23.attn_v.bias", "offset": 311419904, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 311420416, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 311499264, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 311502848, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 313954304, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 316950528, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 319946752, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 319950336, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 319950848, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.3.attn_output.weight", "offset": 320029696, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.3.attn_q.bias", "offset": 320581632, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 320585216, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.3.attn_v.bias", "offset": 321137152, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 321137664, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 321216512, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 321220096, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 323671552, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 326667776, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 329664000, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 329667584, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 329668096, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.4.attn_output.weight", "offset": 329746944, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.4.attn_q.bias", "offset": 330298880, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 330302464, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.4.attn_v.bias", "offset": 330854400, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 330854912, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 330933760, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 330937344, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 333388800, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 336385024, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 339381248, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 339384832, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 339385344, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.5.attn_output.weight", "offset": 339464192, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.5.attn_q.bias", "offset": 340016128, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 340019712, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.5.attn_v.bias", "offset": 340571648, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 340572160, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 340651008, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 340654592, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 343106048, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 346102272, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 349098496, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 349102080, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 349102592, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 349181440, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.6.attn_q.bias", "offset": 349733376, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 349736960, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.6.attn_v.bias", "offset": 350288896, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 350289408, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.7.attn_norm.weight", "offset": 350368256, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 350371840, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 352823296, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 355819520, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 358815744, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 358819328, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 358819840, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.7.attn_output.weight", "offset": 358898688, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.7.attn_q.bias", "offset": 359450624, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 359454208, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.7.attn_v.bias", "offset": 360006144, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 360006656, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 360085504, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 360089088, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 362540544, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 365536768, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 368532992, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 368536576, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 368537088, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.8.attn_output.weight", "offset": 368615936, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.8.attn_q.bias", "offset": 369167872, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 369171456, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.8.attn_v.bias", "offset": 369723392, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 369723904, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 369802752, "shape": [ 896 ], "size": 78848, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 369806336, "shape": [ 4864, 896 ], "size": 3584, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 372257792, "shape": [ 896, 4864 ], "size": 2451456, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 375254016, "shape": [ 896, 4864 ], "size": 2996224, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 378250240, "shape": [ 896 ], "size": 2996224, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 378253824, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 378254336, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "blk.9.attn_output.weight", "offset": 378333184, "shape": [ 896, 896 ], "size": 78848, "type": 6 }, { "name": "blk.9.attn_q.bias", "offset": 378885120, "shape": [ 896 ], "size": 551936, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 378888704, "shape": [ 896, 896 ], "size": 3584, "type": 6 }, { "name": "blk.9.attn_v.bias", "offset": 379440640, "shape": [ 128 ], "size": 551936, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 379441152, "shape": [ 896, 128 ], "size": 512, "type": 6 }, { "name": "output_norm.weight", "offset": 379520000, "shape": [ 896 ], "size": 78848, "type": 0 } ], "version": 3 }