qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 18, "general.name": "Qwen2-0.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 14, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 24, "qwen2.context_length": 32768, "qwen2.embedding_length": 896, "qwen2.feed_forward_length": 4864, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 494032768, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 144643072, "shape": [ 896 ], "size": 144643072, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 144646656, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 148221696, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 152852224, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 157482752, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 157486336, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 157486848, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.0.attn_output.weight", "offset": 157608704, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.0.attn_q.bias", "offset": 158461696, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 158465280, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.0.attn_v.bias", "offset": 159318272, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 159318784, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 159440640, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 159444224, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 163019264, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 167649792, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 172280320, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 172283904, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 172284416, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.1.attn_output.weight", "offset": 172406272, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.1.attn_q.bias", "offset": 173259264, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 173262848, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.1.attn_v.bias", "offset": 174115840, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 174116352, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 174238208, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 174241792, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 177816832, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 182447360, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 187077888, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 187081472, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 187081984, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.10.attn_output.weight", "offset": 187203840, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.10.attn_q.bias", "offset": 188056832, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 188060416, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.10.attn_v.bias", "offset": 188913408, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 188913920, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 189035776, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 189039360, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.11.ffn_gate.weight", "offset": 192614400, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 197244928, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 201875456, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 201879040, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 201879552, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.11.attn_output.weight", "offset": 202001408, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.11.attn_q.bias", "offset": 202854400, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 202857984, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.11.attn_v.bias", "offset": 203710976, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 203711488, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 203833344, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 203836928, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.12.ffn_gate.weight", "offset": 207411968, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 212042496, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 216673024, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 216676608, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 216677120, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.12.attn_output.weight", "offset": 216798976, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.12.attn_q.bias", "offset": 217651968, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 217655552, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.12.attn_v.bias", "offset": 218508544, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 218509056, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 218630912, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 218634496, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 222209536, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 226840064, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 231470592, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 231474176, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 231474688, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.13.attn_output.weight", "offset": 231596544, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.13.attn_q.bias", "offset": 232449536, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 232453120, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.13.attn_v.bias", "offset": 233306112, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 233306624, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 233428480, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 233432064, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.14.ffn_gate.weight", "offset": 237007104, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 241637632, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 246268160, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 246271744, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 246272256, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.14.attn_output.weight", "offset": 246394112, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.14.attn_q.bias", "offset": 247247104, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 247250688, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.14.attn_v.bias", "offset": 248103680, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 248104192, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 248226048, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 248229632, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.15.ffn_gate.weight", "offset": 251804672, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 256435200, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 261065728, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 261069312, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 261069824, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.15.attn_output.weight", "offset": 261191680, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.15.attn_q.bias", "offset": 262044672, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 262048256, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.15.attn_v.bias", "offset": 262901248, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 262901760, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 263023616, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 263027200, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 266602240, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 271232768, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 275863296, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 275866880, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 275867392, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.16.attn_output.weight", "offset": 275989248, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.16.attn_q.bias", "offset": 276842240, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 276845824, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.16.attn_v.bias", "offset": 277698816, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 277699328, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 277821184, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 277824768, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.17.ffn_gate.weight", "offset": 281399808, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 286030336, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 290660864, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 290664448, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 290664960, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.17.attn_output.weight", "offset": 290786816, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.17.attn_q.bias", "offset": 291639808, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 291643392, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.17.attn_v.bias", "offset": 292496384, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 292496896, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 292618752, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 292622336, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.18.ffn_gate.weight", "offset": 296197376, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 300827904, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 305458432, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 305462016, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 305462528, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.18.attn_output.weight", "offset": 305584384, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.18.attn_q.bias", "offset": 306437376, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 306440960, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.18.attn_v.bias", "offset": 307293952, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 307294464, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 307416320, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 307419904, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 310994944, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 315625472, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 320256000, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 320259584, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 320260096, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.19.attn_output.weight", "offset": 320381952, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.19.attn_q.bias", "offset": 321234944, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 321238528, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.19.attn_v.bias", "offset": 322091520, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 322092032, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 322213888, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 322217472, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.2.ffn_gate.weight", "offset": 325792512, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 330423040, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 335053568, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 335057152, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 335057664, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.2.attn_output.weight", "offset": 335179520, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.2.attn_q.bias", "offset": 336032512, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 336036096, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.2.attn_v.bias", "offset": 336889088, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 336889600, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 337011456, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 337015040, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.20.ffn_gate.weight", "offset": 340590080, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 345220608, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 349851136, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 349854720, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 349855232, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.20.attn_output.weight", "offset": 349977088, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.20.attn_q.bias", "offset": 350830080, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 350833664, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.20.attn_v.bias", "offset": 351686656, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 351687168, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 351809024, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 351812608, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 355387648, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 360018176, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 364648704, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 364652288, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 364652800, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 364774656, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.21.attn_q.bias", "offset": 365627648, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 365631232, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.21.attn_v.bias", "offset": 366484224, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 366484736, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.22.attn_norm.weight", "offset": 366606592, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 366610176, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.22.ffn_gate.weight", "offset": 370185216, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 374815744, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 379446272, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 379449856, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 379450368, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.22.attn_output.weight", "offset": 379572224, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.22.attn_q.bias", "offset": 380425216, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 380428800, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.22.attn_v.bias", "offset": 381281792, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 381282304, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 381404160, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 381407744, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.23.ffn_gate.weight", "offset": 384982784, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 389613312, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 394243840, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 394247424, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 394247936, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.23.attn_output.weight", "offset": 394369792, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.23.attn_q.bias", "offset": 395222784, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 395226368, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.23.attn_v.bias", "offset": 396079360, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 396079872, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 396201728, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 396205312, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.3.ffn_gate.weight", "offset": 399780352, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 404410880, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 409041408, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 409044992, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 409045504, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.3.attn_output.weight", "offset": 409167360, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.3.attn_q.bias", "offset": 410020352, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 410023936, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.3.attn_v.bias", "offset": 410876928, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 410877440, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 410999296, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 411002880, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.4.ffn_gate.weight", "offset": 414577920, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 419208448, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 423838976, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 423842560, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 423843072, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.4.attn_output.weight", "offset": 423964928, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.4.attn_q.bias", "offset": 424817920, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 424821504, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.4.attn_v.bias", "offset": 425674496, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 425675008, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 425796864, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 425800448, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 429375488, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 434006016, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 438636544, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 438640128, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 438640640, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.5.attn_output.weight", "offset": 438762496, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.5.attn_q.bias", "offset": 439615488, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 439619072, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.5.attn_v.bias", "offset": 440472064, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 440472576, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 440594432, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 440598016, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 444173056, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 448803584, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 453434112, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 453437696, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 453438208, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 453560064, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.6.attn_q.bias", "offset": 454413056, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 454416640, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.6.attn_v.bias", "offset": 455269632, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 455270144, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.7.attn_norm.weight", "offset": 455392000, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 455395584, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 458970624, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 463601152, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 468231680, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 468235264, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 468235776, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.7.attn_output.weight", "offset": 468357632, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.7.attn_q.bias", "offset": 469210624, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 469214208, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.7.attn_v.bias", "offset": 470067200, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 470067712, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 470189568, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 470193152, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 473768192, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 478398720, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 483029248, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 483032832, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 483033344, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.8.attn_output.weight", "offset": 483155200, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.8.attn_q.bias", "offset": 484008192, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 484011776, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.8.attn_v.bias", "offset": 484864768, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 484865280, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 484987136, "shape": [ 896 ], "size": 121856, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 484990720, "shape": [ 4864, 896 ], "size": 3584, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 488565760, "shape": [ 896, 4864 ], "size": 3575040, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 493196288, "shape": [ 896, 4864 ], "size": 4630528, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 497826816, "shape": [ 896 ], "size": 4630528, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 497830400, "shape": [ 128 ], "size": 3584, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 497830912, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "blk.9.attn_output.weight", "offset": 497952768, "shape": [ 896, 896 ], "size": 121856, "type": 8 }, { "name": "blk.9.attn_q.bias", "offset": 498805760, "shape": [ 896 ], "size": 852992, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 498809344, "shape": [ 896, 896 ], "size": 3584, "type": 8 }, { "name": "blk.9.attn_v.bias", "offset": 499662336, "shape": [ 128 ], "size": 852992, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 499662848, "shape": [ 896, 128 ], "size": 512, "type": 8 }, { "name": "output_norm.weight", "offset": 499784704, "shape": [ 896 ], "size": 121856, "type": 0 } ], "version": 3 }