qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 12, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 200907264, "shape": [ 1536, 8960 ], "size": 9461760, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 206820864, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 212734464, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 212740608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 212741632, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.0.attn_output.weight", "offset": 212910592, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 214237696, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 214243840, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.0.attn_v.bias", "offset": 215257600, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 215258624, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 215528960, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 215535104, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.1.ffn_gate.weight", "offset": 223276544, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 229190144, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 235103744, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 235109888, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 235110912, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.1.attn_output.weight", "offset": 235279872, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 236606976, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 236613120, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.1.attn_v.bias", "offset": 237626880, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 237627904, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 237898240, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 237904384, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.10.ffn_gate.weight", "offset": 245645824, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 251559424, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 257473024, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 257479168, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 257480192, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.10.attn_output.weight", "offset": 257649152, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 258976256, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 258982400, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.10.attn_v.bias", "offset": 259996160, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 259997184, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 260218368, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 260224512, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.11.ffn_gate.weight", "offset": 267965952, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 273879552, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 279793152, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 279799296, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 279800320, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.11.attn_output.weight", "offset": 279969280, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 281296384, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 281302528, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.11.attn_v.bias", "offset": 282316288, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 282317312, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 282538496, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 282544640, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 290286080, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 296199680, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 302113280, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 302119424, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 302120448, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.12.attn_output.weight", "offset": 302289408, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 303616512, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 303622656, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.12.attn_v.bias", "offset": 304636416, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 304637440, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 304858624, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 304864768, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.13.ffn_gate.weight", "offset": 312606208, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 318519808, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 324433408, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 324439552, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 324440576, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.13.attn_output.weight", "offset": 324609536, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 325936640, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 325942784, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.13.attn_v.bias", "offset": 326956544, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 326957568, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 327178752, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 327184896, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 334926336, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 340839936, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 346753536, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 346759680, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 346760704, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.14.attn_output.weight", "offset": 346929664, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 348256768, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 348262912, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.14.attn_v.bias", "offset": 349276672, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 349277696, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 349498880, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 349505024, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 357246464, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 363160064, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 369073664, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 369079808, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 369080832, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.15.attn_output.weight", "offset": 369249792, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 370576896, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 370583040, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.15.attn_v.bias", "offset": 371596800, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 371597824, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 371819008, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 371825152, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.16.ffn_gate.weight", "offset": 379566592, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 385480192, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 391393792, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 391399936, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 391400960, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.16.attn_output.weight", "offset": 391569920, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 392897024, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 392903168, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.16.attn_v.bias", "offset": 393916928, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 393917952, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 394139136, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 394145280, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 401886720, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 407800320, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 413713920, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 413720064, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 413721088, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.17.attn_output.weight", "offset": 413890048, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 415217152, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 415223296, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.17.attn_v.bias", "offset": 416237056, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 416238080, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 416459264, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 416465408, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 424206848, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 430120448, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 436034048, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 436040192, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 436041216, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.18.attn_output.weight", "offset": 436210176, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 437537280, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 437543424, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.18.attn_v.bias", "offset": 438557184, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 438558208, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 438779392, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 438785536, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.19.ffn_gate.weight", "offset": 446526976, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 452440576, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 458354176, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 458360320, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 458361344, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.19.attn_output.weight", "offset": 458530304, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 459857408, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 459863552, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.19.attn_v.bias", "offset": 460877312, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 460878336, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 461099520, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 461105664, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 468847104, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 474760704, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 480674304, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 480680448, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 480681472, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.2.attn_output.weight", "offset": 480850432, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 482177536, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 482183680, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.2.attn_v.bias", "offset": 483197440, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 483198464, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 483419648, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 483425792, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 491167232, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 497080832, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 502994432, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 503000576, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 503001600, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.20.attn_output.weight", "offset": 503170560, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 504497664, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 504503808, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.20.attn_v.bias", "offset": 505517568, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 505518592, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 505739776, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 505745920, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.21.ffn_gate.weight", "offset": 513487360, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 519400960, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 525314560, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 525320704, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 525321728, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 525490688, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 526817792, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 526823936, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.21.attn_v.bias", "offset": 527837696, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 527838720, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.22.attn_norm.weight", "offset": 528059904, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 528066048, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 535807488, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 541721088, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 547634688, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 547640832, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 547641856, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.22.attn_output.weight", "offset": 547810816, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 549137920, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 549144064, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.22.attn_v.bias", "offset": 550157824, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 550158848, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 550380032, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 550386176, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 558127616, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 564041216, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 569954816, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 569960960, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 569961984, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.23.attn_output.weight", "offset": 570130944, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 571458048, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 571464192, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.23.attn_v.bias", "offset": 572477952, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 572478976, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 572700160, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 572706304, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.24.ffn_gate.weight", "offset": 580447744, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 586361344, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 592274944, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 592281088, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 592282112, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.24.attn_output.weight", "offset": 592451072, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 593778176, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 593784320, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.24.attn_v.bias", "offset": 594798080, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 594799104, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 595020288, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 595026432, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 602767872, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 608681472, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 614595072, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 614601216, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 614602240, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.25.attn_output.weight", "offset": 614771200, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 616098304, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 616104448, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.25.attn_v.bias", "offset": 617118208, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 617119232, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 617340416, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 617346560, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 625088000, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 631001600, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 636915200, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 636921344, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 636922368, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.26.attn_output.weight", "offset": 637091328, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 638418432, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 638424576, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.26.attn_v.bias", "offset": 639438336, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 639439360, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 639660544, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 639666688, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.27.ffn_gate.weight", "offset": 647408128, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 653321728, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 659235328, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 659241472, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 659242496, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.27.attn_output.weight", "offset": 659411456, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 660738560, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 660744704, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.27.attn_v.bias", "offset": 661758464, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 661759488, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "Їy\ufffd\ufffd7\rB\ufffdT\ufffd\u0014pB\ufffd\ufffd0\u0010\ufffd\u0026\u0018\ufffd", "offset": 661980672, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 661986816, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 669728256, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 675641856, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 681555456, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 681561600, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 681562624, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.3.attn_output.weight", "offset": 681731584, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 683058688, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 683064832, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.3.attn_v.bias", "offset": 684078592, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 684079616, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 684300800, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 684306944, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 692048384, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 697961984, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 703875584, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 703881728, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 703882752, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.4.attn_output.weight", "offset": 704051712, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 705378816, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 705384960, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.4.attn_v.bias", "offset": 706398720, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 706399744, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 706620928, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 706627072, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.5.ffn_gate.weight", "offset": 714368512, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 720282112, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 726195712, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 726201856, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 726202880, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.5.attn_output.weight", "offset": 726371840, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 727698944, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 727705088, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.5.attn_v.bias", "offset": 728718848, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 728719872, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 728941056, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 728947200, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.6.ffn_gate.weight", "offset": 736688640, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 742602240, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 748515840, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 748521984, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 748523008, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 748691968, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 750019072, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 750025216, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.6.attn_v.bias", "offset": 751038976, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 751040000, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.7.attn_norm.weight", "offset": 751261184, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 751267328, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.7.ffn_gate.weight", "offset": 759008768, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 764922368, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 770835968, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 770842112, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 770843136, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.7.attn_output.weight", "offset": 771012096, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 772339200, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 772345344, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.7.attn_v.bias", "offset": 773359104, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 773360128, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 773581312, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 773587456, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.8.ffn_gate.weight", "offset": 781328896, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 787242496, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 793156096, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 793162240, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 793163264, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.8.attn_output.weight", "offset": 793332224, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 794659328, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 794665472, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.8.attn_v.bias", "offset": 795679232, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 795680256, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 795901440, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 795907584, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.9.ffn_gate.weight", "offset": 803649024, "shape": [ 1536, 8960 ], "size": 7741440, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 809562624, "shape": [ 1536, 8960 ], "size": 5913600, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 815476224, "shape": [ 1536 ], "size": 5913600, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 815482368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 815483392, "shape": [ 1536, 256 ], "size": 1024, "type": 11 }, { "name": "blk.9.attn_output.weight", "offset": 815652352, "shape": [ 1536, 1536 ], "size": 168960, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 816979456, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 816985600, "shape": [ 1536, 1536 ], "size": 6144, "type": 11 }, { "name": "blk.9.attn_v.bias", "offset": 817999360, "shape": [ 256 ], "size": 1013760, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 818000384, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "output_norm.weight", "offset": 818221568, "shape": [ 1536 ], "size": 221184, "type": 0 } ], "version": 3 }