qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 13, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 147642880, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.0.ffn_up.weight", "offset": 150094336, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.0.ffn_norm.weight", "offset": 152545792, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 152549376, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 152549888, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.0.attn_output.weight", "offset": 152614400, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.0.attn_q.bias", "offset": 153216512, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 153220096, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.0.attn_v.bias", "offset": 153671680, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 153672192, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 153758208, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 153761792, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 156758016, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.1.ffn_up.weight", "offset": 159209472, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.1.ffn_norm.weight", "offset": 161660928, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 161664512, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 161665024, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.1.attn_output.weight", "offset": 161729536, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.1.attn_q.bias", "offset": 162331648, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 162335232, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.1.attn_v.bias", "offset": 162786816, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 162787328, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 162873344, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 162876928, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 165873152, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.10.ffn_up.weight", "offset": 168324608, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.10.ffn_norm.weight", "offset": 170776064, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 170779648, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 170780160, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.10.attn_output.weight", "offset": 170844672, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.10.attn_q.bias", "offset": 171446784, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 171450368, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.10.attn_v.bias", "offset": 171901952, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 171902464, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 171988480, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 171992064, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.11.ffn_gate.weight", "offset": 174988288, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.11.ffn_up.weight", "offset": 177439744, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.11.ffn_norm.weight", "offset": 179891200, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 179894784, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 179895296, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.11.attn_output.weight", "offset": 179959808, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.11.attn_q.bias", "offset": 180561920, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 180565504, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.11.attn_v.bias", "offset": 181017088, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 181017600, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 181103616, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 181107200, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 184103424, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.12.ffn_up.weight", "offset": 186554880, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.12.ffn_norm.weight", "offset": 189006336, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 189009920, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 189010432, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.12.attn_output.weight", "offset": 189074944, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.12.attn_q.bias", "offset": 189677056, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 189680640, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.12.attn_v.bias", "offset": 190132224, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 190132736, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 190218752, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 190222336, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 193218560, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.13.ffn_up.weight", "offset": 195670016, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.13.ffn_norm.weight", "offset": 198121472, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 198125056, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 198125568, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.13.attn_output.weight", "offset": 198190080, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.13.attn_q.bias", "offset": 198792192, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 198795776, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.13.attn_v.bias", "offset": 199247360, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 199247872, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 199333888, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 199337472, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 202333696, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.14.ffn_up.weight", "offset": 204785152, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.14.ffn_norm.weight", "offset": 207236608, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 207240192, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 207240704, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.14.attn_output.weight", "offset": 207305216, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.14.attn_q.bias", "offset": 207907328, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 207910912, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.14.attn_v.bias", "offset": 208362496, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 208363008, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 208449024, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 208452608, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 211448832, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.15.ffn_up.weight", "offset": 213900288, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.15.ffn_norm.weight", "offset": 216351744, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 216355328, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 216355840, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.15.attn_output.weight", "offset": 216420352, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.15.attn_q.bias", "offset": 217022464, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 217026048, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.15.attn_v.bias", "offset": 217477632, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 217478144, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 217564160, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 217567744, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 220563968, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.16.ffn_up.weight", "offset": 223015424, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.16.ffn_norm.weight", "offset": 225466880, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 225470464, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 225470976, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.16.attn_output.weight", "offset": 225535488, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.16.attn_q.bias", "offset": 226137600, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 226141184, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.16.attn_v.bias", "offset": 226592768, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 226593280, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 226679296, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 226682880, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 229679104, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.17.ffn_up.weight", "offset": 232130560, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.17.ffn_norm.weight", "offset": 234582016, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 234585600, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 234586112, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.17.attn_output.weight", "offset": 234650624, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.17.attn_q.bias", "offset": 235252736, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 235256320, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.17.attn_v.bias", "offset": 235707904, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 235708416, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 235794432, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 235798016, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 238794240, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.18.ffn_up.weight", "offset": 241245696, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.18.ffn_norm.weight", "offset": 243697152, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 243700736, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 243701248, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.18.attn_output.weight", "offset": 243765760, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.18.attn_q.bias", "offset": 244367872, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 244371456, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.18.attn_v.bias", "offset": 244823040, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 244823552, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 244909568, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 244913152, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 247909376, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.19.ffn_up.weight", "offset": 250360832, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.19.ffn_norm.weight", "offset": 252812288, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 252815872, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 252816384, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.19.attn_output.weight", "offset": 252880896, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.19.attn_q.bias", "offset": 253483008, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 253486592, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.19.attn_v.bias", "offset": 253938176, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 253938688, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 254024704, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 254028288, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 257024512, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.2.ffn_up.weight", "offset": 259475968, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.2.ffn_norm.weight", "offset": 261927424, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 261931008, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 261931520, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.2.attn_output.weight", "offset": 261996032, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.2.attn_q.bias", "offset": 262598144, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 262601728, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.2.attn_v.bias", "offset": 263053312, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 263053824, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 263139840, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 263143424, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 266139648, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.20.ffn_up.weight", "offset": 268591104, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.20.ffn_norm.weight", "offset": 271042560, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 271046144, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 271046656, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.20.attn_output.weight", "offset": 271111168, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.20.attn_q.bias", "offset": 271713280, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 271716864, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.20.attn_v.bias", "offset": 272168448, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 272168960, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 272254976, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 272258560, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 275254784, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.21.ffn_up.weight", "offset": 277706240, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.21.ffn_norm.weight", "offset": 280157696, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 280161280, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 280161792, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.21.attn_output.weight", "offset": 280226304, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.21.attn_q.bias", "offset": 280828416, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 280832000, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.21.attn_v.bias", "offset": 281283584, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 281284096, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.22.attn_norm.weight", "offset": 281370112, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 281373696, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 284369920, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.22.ffn_up.weight", "offset": 286821376, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.22.ffn_norm.weight", "offset": 289272832, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 289276416, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 289276928, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.22.attn_output.weight", "offset": 289341440, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.22.attn_q.bias", "offset": 289943552, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 289947136, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.22.attn_v.bias", "offset": 290398720, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 290399232, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 290485248, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 290488832, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 293485056, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.23.ffn_up.weight", "offset": 295936512, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.23.ffn_norm.weight", "offset": 298387968, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 298391552, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 298392064, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.23.attn_output.weight", "offset": 298456576, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.23.attn_q.bias", "offset": 299058688, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 299062272, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.23.attn_v.bias", "offset": 299513856, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 299514368, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 299600384, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 299603968, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 302600192, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.3.ffn_up.weight", "offset": 305051648, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.3.ffn_norm.weight", "offset": 307503104, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 307506688, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 307507200, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.3.attn_output.weight", "offset": 307571712, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.3.attn_q.bias", "offset": 308173824, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 308177408, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.3.attn_v.bias", "offset": 308628992, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 308629504, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 308715520, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 308719104, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 311715328, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.4.ffn_up.weight", "offset": 314166784, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.4.ffn_norm.weight", "offset": 316618240, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 316621824, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 316622336, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.4.attn_output.weight", "offset": 316686848, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.4.attn_q.bias", "offset": 317288960, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 317292544, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.4.attn_v.bias", "offset": 317744128, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 317744640, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 317830656, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 317834240, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 320830464, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.5.ffn_up.weight", "offset": 323281920, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.5.ffn_norm.weight", "offset": 325733376, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 325736960, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 325737472, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.5.attn_output.weight", "offset": 325801984, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.5.attn_q.bias", "offset": 326404096, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 326407680, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.5.attn_v.bias", "offset": 326859264, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 326859776, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 326945792, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 326949376, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 329945600, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.6.ffn_up.weight", "offset": 332397056, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.6.ffn_norm.weight", "offset": 334848512, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 334852096, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 334852608, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.6.attn_output.weight", "offset": 334917120, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.6.attn_q.bias", "offset": 335519232, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 335522816, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.6.attn_v.bias", "offset": 335974400, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 335974912, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.7.attn_norm.weight", "offset": 336060928, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 336064512, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 339060736, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.7.ffn_up.weight", "offset": 341512192, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.7.ffn_norm.weight", "offset": 343963648, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 343967232, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 343967744, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.7.attn_output.weight", "offset": 344032256, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.7.attn_q.bias", "offset": 344634368, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 344637952, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.7.attn_v.bias", "offset": 345089536, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 345090048, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 345176064, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 345179648, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 348175872, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.8.ffn_up.weight", "offset": 350627328, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.8.ffn_norm.weight", "offset": 353078784, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 353082368, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 353082880, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.8.attn_output.weight", "offset": 353147392, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.8.attn_q.bias", "offset": 353749504, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 353753088, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.8.attn_v.bias", "offset": 354204672, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 354205184, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 354291200, "shape": [ 896 ], "size": 86016, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 354294784, "shape": [ 4864, 896 ], "size": 3584, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 357291008, "shape": [ 896, 4864 ], "size": 2996224, "type": 20 }, { "name": "blk.9.ffn_up.weight", "offset": 359742464, "shape": [ 896, 4864 ], "size": 2451456, "type": 20 }, { "name": "blk.9.ffn_norm.weight", "offset": 362193920, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 362197504, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 362198016, "shape": [ 896, 128 ], "size": 512, "type": 20 }, { "name": "blk.9.attn_output.weight", "offset": 362262528, "shape": [ 896, 896 ], "size": 64512, "type": 7 }, { "name": "blk.9.attn_q.bias", "offset": 362864640, "shape": [ 896 ], "size": 602112, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 362868224, "shape": [ 896, 896 ], "size": 3584, "type": 20 }, { "name": "blk.9.attn_v.bias", "offset": 363319808, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 363320320, "shape": [ 896, 128 ], "size": 512, "type": 7 }, { "name": "output_norm.weight", "offset": 363406336, "shape": [ 896 ], "size": 86016, "type": 0 } ], "version": 3 }