qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 2, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.0.ffn_gate.weight", "offset": 147098112, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 149549568, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 152001024, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 152004608, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 152005120, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.0.attn_output.weight", "offset": 152069632, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.0.attn_q.bias", "offset": 152521216, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 152524800, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.0.attn_v.bias", "offset": 152976384, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 152976896, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 153041408, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 153044992, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.1.ffn_gate.weight", "offset": 155496448, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 157947904, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 160399360, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 160402944, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 160403456, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.1.attn_output.weight", "offset": 160467968, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.1.attn_q.bias", "offset": 160919552, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 160923136, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.1.attn_v.bias", "offset": 161374720, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 161375232, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 161439744, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 161443328, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.10.ffn_gate.weight", "offset": 163894784, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 166346240, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 168797696, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 168801280, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 168801792, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.10.attn_output.weight", "offset": 168866304, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.10.attn_q.bias", "offset": 169317888, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 169321472, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.10.attn_v.bias", "offset": 169773056, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 169773568, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 169838080, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 169841664, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.11.ffn_gate.weight", "offset": 172293120, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 174744576, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 177196032, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 177199616, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 177200128, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.11.attn_output.weight", "offset": 177264640, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.11.attn_q.bias", "offset": 177716224, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 177719808, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.11.attn_v.bias", "offset": 178171392, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 178171904, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 178236416, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 178240000, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.12.ffn_gate.weight", "offset": 180691456, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 183142912, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 185594368, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 185597952, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 185598464, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.12.attn_output.weight", "offset": 185662976, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.12.attn_q.bias", "offset": 186114560, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 186118144, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.12.attn_v.bias", "offset": 186569728, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 186570240, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 186634752, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 186638336, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.13.ffn_gate.weight", "offset": 189089792, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 191541248, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 193992704, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 193996288, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 193996800, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.13.attn_output.weight", "offset": 194061312, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.13.attn_q.bias", "offset": 194512896, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 194516480, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.13.attn_v.bias", "offset": 194968064, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 194968576, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 195033088, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 195036672, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.14.ffn_gate.weight", "offset": 197488128, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 199939584, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 202391040, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 202394624, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 202395136, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.14.attn_output.weight", "offset": 202459648, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.14.attn_q.bias", "offset": 202911232, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 202914816, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.14.attn_v.bias", "offset": 203366400, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 203366912, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 203431424, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 203435008, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.15.ffn_gate.weight", "offset": 205886464, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 208337920, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 210789376, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 210792960, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 210793472, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.15.attn_output.weight", "offset": 210857984, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.15.attn_q.bias", "offset": 211309568, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 211313152, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.15.attn_v.bias", "offset": 211764736, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 211765248, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 211829760, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 211833344, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.16.ffn_gate.weight", "offset": 214284800, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 216736256, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 219187712, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 219191296, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 219191808, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.16.attn_output.weight", "offset": 219256320, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.16.attn_q.bias", "offset": 219707904, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 219711488, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.16.attn_v.bias", "offset": 220163072, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 220163584, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 220228096, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 220231680, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.17.ffn_gate.weight", "offset": 222683136, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 225134592, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 227586048, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 227589632, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 227590144, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.17.attn_output.weight", "offset": 227654656, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.17.attn_q.bias", "offset": 228106240, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 228109824, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.17.attn_v.bias", "offset": 228561408, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 228561920, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 228626432, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 228630016, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.18.ffn_gate.weight", "offset": 231081472, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 233532928, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 235984384, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 235987968, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 235988480, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.18.attn_output.weight", "offset": 236052992, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.18.attn_q.bias", "offset": 236504576, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 236508160, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.18.attn_v.bias", "offset": 236959744, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 236960256, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 237024768, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 237028352, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.19.ffn_gate.weight", "offset": 239479808, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 241931264, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 244382720, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 244386304, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 244386816, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.19.attn_output.weight", "offset": 244451328, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.19.attn_q.bias", "offset": 244902912, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 244906496, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.19.attn_v.bias", "offset": 245358080, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 245358592, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 245423104, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 245426688, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.2.ffn_gate.weight", "offset": 247878144, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 250329600, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 252781056, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 252784640, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 252785152, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.2.attn_output.weight", "offset": 252849664, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.2.attn_q.bias", "offset": 253301248, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 253304832, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.2.attn_v.bias", "offset": 253756416, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 253756928, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 253821440, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 253825024, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.20.ffn_gate.weight", "offset": 256276480, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 258727936, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 261179392, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 261182976, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 261183488, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.20.attn_output.weight", "offset": 261248000, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.20.attn_q.bias", "offset": 261699584, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 261703168, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.20.attn_v.bias", "offset": 262154752, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 262155264, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 262219776, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 262223360, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.21.ffn_gate.weight", "offset": 264674816, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 267126272, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 269577728, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 269581312, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 269581824, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 269646336, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.21.attn_q.bias", "offset": 270097920, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 270101504, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.21.attn_v.bias", "offset": 270553088, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 270553600, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 270618112, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 270621696, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.22.ffn_gate.weight", "offset": 273073152, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 275524608, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 277976064, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 277979648, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 277980160, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.22.attn_output.weight", "offset": 278044672, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.22.attn_q.bias", "offset": 278496256, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 278499840, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.22.attn_v.bias", "offset": 278951424, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 278951936, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 279016448, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 279020032, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.23.ffn_gate.weight", "offset": 281471488, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 283922944, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 286374400, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 286377984, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 286378496, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.23.attn_output.weight", "offset": 286443008, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.23.attn_q.bias", "offset": 286894592, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 286898176, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.23.attn_v.bias", "offset": 287349760, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 287350272, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 287414784, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 287418368, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.3.ffn_gate.weight", "offset": 289869824, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 292321280, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 294772736, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 294776320, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 294776832, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.3.attn_output.weight", "offset": 294841344, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.3.attn_q.bias", "offset": 295292928, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 295296512, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.3.attn_v.bias", "offset": 295748096, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 295748608, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 295813120, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 295816704, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.4.ffn_gate.weight", "offset": 298268160, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 300719616, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 303171072, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 303174656, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 303175168, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.4.attn_output.weight", "offset": 303239680, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.4.attn_q.bias", "offset": 303691264, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 303694848, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.4.attn_v.bias", "offset": 304146432, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 304146944, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 304211456, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 304215040, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.5.ffn_gate.weight", "offset": 306666496, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 309117952, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 311569408, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 311572992, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 311573504, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.5.attn_output.weight", "offset": 311638016, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.5.attn_q.bias", "offset": 312089600, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 312093184, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.5.attn_v.bias", "offset": 312544768, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 312545280, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 312609792, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 312613376, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.6.ffn_gate.weight", "offset": 315064832, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 317516288, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 319967744, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 319971328, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 319971840, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 320036352, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.6.attn_q.bias", "offset": 320487936, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 320491520, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.6.attn_v.bias", "offset": 320943104, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 320943616, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 321008128, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 321011712, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.7.ffn_gate.weight", "offset": 323463168, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 325914624, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 328366080, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 328369664, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 328370176, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.7.attn_output.weight", "offset": 328434688, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.7.attn_q.bias", "offset": 328886272, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 328889856, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.7.attn_v.bias", "offset": 329341440, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 329341952, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 329406464, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 329410048, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.8.ffn_gate.weight", "offset": 331861504, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 334312960, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 336764416, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 336768000, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 336768512, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.8.attn_output.weight", "offset": 336833024, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.8.attn_q.bias", "offset": 337284608, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 337288192, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.8.attn_v.bias", "offset": 337739776, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 337740288, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 337804800, "shape": [ 896 ], "size": 64512, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 337808384, "shape": [ 4864, 896 ], "size": 3584, "type": 2 }, { "name": "blk.9.ffn_gate.weight", "offset": 340259840, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 342711296, "shape": [ 896, 4864 ], "size": 2451456, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 345162752, "shape": [ 896 ], "size": 2451456, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 345166336, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 345166848, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "blk.9.attn_output.weight", "offset": 345231360, "shape": [ 896, 896 ], "size": 64512, "type": 2 }, { "name": "blk.9.attn_q.bias", "offset": 345682944, "shape": [ 896 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 345686528, "shape": [ 896, 896 ], "size": 3584, "type": 2 }, { "name": "blk.9.attn_v.bias", "offset": 346138112, "shape": [ 128 ], "size": 451584, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 346138624, "shape": [ 896, 128 ], "size": 512, "type": 2 }, { "name": "output_norm.weight", "offset": 346203136, "shape": [ 896 ], "size": 64512, "type": 0 } ], "version": 3 }