qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 15, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.0.ffn_gate.weight", "offset": 202735104, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.0.ffn_up.weight", "offset": 210476544, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 218217984, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 218224128, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 218225152, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.0.attn_output.weight", "offset": 218446336, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.0.attn_q.bias", "offset": 219773440, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 219779584, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.0.attn_v.bias", "offset": 221106688, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 221107712, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 221430272, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 221436416, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.1.ffn_gate.weight", "offset": 232726016, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.1.ffn_up.weight", "offset": 240467456, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 248208896, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 248215040, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 248216064, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.1.attn_output.weight", "offset": 248437248, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.1.attn_q.bias", "offset": 249764352, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 249770496, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.1.attn_v.bias", "offset": 251097600, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 251098624, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 251421184, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 251427328, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.10.ffn_gate.weight", "offset": 262716928, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 270458368, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 278199808, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 278205952, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 278206976, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.10.attn_output.weight", "offset": 278428160, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.10.attn_q.bias", "offset": 279755264, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 279761408, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.10.attn_v.bias", "offset": 281088512, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 281089536, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 281412096, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 281418240, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "\u0000\u0000\u0000blk.11.attn_k.weigh", "offset": 289159680, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 296901120, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 304642560, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 304648704, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 304649728, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.11.attn_output.weight", "offset": 304870912, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.11.attn_q.bias", "offset": 306198016, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 306204160, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.11.attn_v.bias", "offset": 307531264, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 307532288, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 307753472, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 307759616, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.12.ffn_gate.weight", "offset": 315501056, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 323242496, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 330983936, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 330990080, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 330991104, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.12.attn_output.weight", "offset": 331212288, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.12.attn_q.bias", "offset": 332539392, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 332545536, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.12.attn_v.bias", "offset": 333872640, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 333873664, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 334094848, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 334100992, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.13.ffn_gate.weight", "offset": 345390592, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 353132032, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 360873472, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 360879616, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 360880640, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.13.attn_output.weight", "offset": 361101824, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.13.attn_q.bias", "offset": 362428928, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 362435072, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.13.attn_v.bias", "offset": 363762176, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 363763200, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 364085760, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 364091904, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.14.ffn_gate.weight", "offset": 371833344, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 379574784, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 387316224, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 387322368, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 387323392, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.14.attn_output.weight", "offset": 387544576, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.14.attn_q.bias", "offset": 388871680, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 388877824, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.14.attn_v.bias", "offset": 390204928, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 390205952, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 390427136, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 390433280, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.15.ffn_gate.weight", "offset": 398174720, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 405916160, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 413657600, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 413663744, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 413664768, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.15.attn_output.weight", "offset": 413885952, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.15.attn_q.bias", "offset": 415213056, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 415219200, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.15.attn_v.bias", "offset": 416546304, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 416547328, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 416768512, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 416774656, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.16.ffn_gate.weight", "offset": 428064256, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 435805696, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 443547136, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 443553280, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 443554304, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.16.attn_output.weight", "offset": 443775488, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.16.attn_q.bias", "offset": 445102592, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 445108736, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.16.attn_v.bias", "offset": 446435840, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 446436864, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 446759424, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 446765568, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.17.ffn_gate.weight", "offset": 454507008, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 462248448, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 469989888, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 469996032, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 469997056, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.17.attn_output.weight", "offset": 470218240, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.17.attn_q.bias", "offset": 471545344, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 471551488, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.17.attn_v.bias", "offset": 472878592, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 472879616, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 473100800, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 473106944, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.18.ffn_gate.weight", "offset": 480848384, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 488589824, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 496331264, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 496337408, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 496338432, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.18.attn_output.weight", "offset": 496559616, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.18.attn_q.bias", "offset": 497886720, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 497892864, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.18.attn_v.bias", "offset": 499219968, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 499220992, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 499442176, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 499448320, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.19.ffn_gate.weight", "offset": 510737920, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 518479360, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 526220800, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 526226944, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 526227968, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.19.attn_output.weight", "offset": 526449152, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.19.attn_q.bias", "offset": 527776256, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 527782400, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.19.attn_v.bias", "offset": 529109504, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 529110528, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 529433088, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 529439232, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.2.ffn_gate.weight", "offset": 537180672, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 544922112, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 552663552, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 552669696, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 552670720, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.2.attn_output.weight", "offset": 552891904, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.2.attn_q.bias", "offset": 554219008, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 554225152, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.2.attn_v.bias", "offset": 555552256, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 555553280, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 555774464, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 555780608, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.20.ffn_gate.weight", "offset": 563522048, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 571263488, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 579004928, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 579011072, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 579012096, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.20.attn_output.weight", "offset": 579233280, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.20.attn_q.bias", "offset": 580560384, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 580566528, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.20.attn_v.bias", "offset": 581893632, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 581894656, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 582115840, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 582121984, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.21.ffn_gate.weight", "offset": 593411584, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 601153024, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 608894464, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 608900608, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 608901632, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 609122816, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.21.attn_q.bias", "offset": 610449920, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 610456064, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.21.attn_v.bias", "offset": 611783168, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 611784192, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.22.attn_norm.weight", "offset": 612106752, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 612112896, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.22.ffn_gate.weight", "offset": 619854336, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 627595776, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 635337216, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 635343360, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 635344384, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.22.attn_output.weight", "offset": 635565568, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.22.attn_q.bias", "offset": 636892672, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 636898816, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.22.attn_v.bias", "offset": 638225920, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 638226944, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 638448128, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 638454272, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.23.ffn_gate.weight", "offset": 646195712, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 653937152, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 661678592, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 661684736, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 661685760, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.23.attn_output.weight", "offset": 661906944, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.23.attn_q.bias", "offset": 663234048, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 663240192, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.23.attn_v.bias", "offset": 664567296, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 664568320, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 664789504, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 664795648, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.24.ffn_gate.weight", "offset": 676085248, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 683826688, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 691568128, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 691574272, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 691575296, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.24.attn_output.weight", "offset": 691796480, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.24.attn_q.bias", "offset": 693123584, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 693129728, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.24.attn_v.bias", "offset": 694456832, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 694457856, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 694780416, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 694786560, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.25.ffn_gate.weight", "offset": 702528000, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 710269440, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 718010880, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 718017024, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 718018048, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.25.attn_output.weight", "offset": 718239232, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.25.attn_q.bias", "offset": 719566336, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 719572480, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.25.attn_v.bias", "offset": 720899584, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 720900608, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 721121792, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 721127936, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.26.ffn_gate.weight", "offset": 728869376, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 736610816, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 744352256, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 744358400, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 744359424, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.26.attn_output.weight", "offset": 744580608, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.26.attn_q.bias", "offset": 745907712, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 745913856, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.26.attn_v.bias", "offset": 747240960, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 747241984, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 747463168, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 747469312, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.27.ffn_gate.weight", "offset": 758758912, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 766500352, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 774241792, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 774247936, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 774248960, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.27.attn_output.weight", "offset": 774470144, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.27.attn_q.bias", "offset": 775797248, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 775803392, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.27.attn_v.bias", "offset": 777130496, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 777131520, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 777454080, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "\u0000\u0000\u0000\u0000z\ufffd8\u0000\u0000\u0000\u0000\u0015\u0000\u0000\u0000\u0000\u0000\u0000\u0000bl", "offset": 777460224, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.3.ffn_gate.weight", "offset": 785201664, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 792943104, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 800684544, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 800690688, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 800691712, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.3.attn_output.weight", "offset": 800912896, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.3.attn_q.bias", "offset": 802240000, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 802246144, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.3.attn_v.bias", "offset": 803573248, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 803574272, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 803795456, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 803801600, "shape": [ 8960, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.4.ffn_gate.weight", "offset": 811543040, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 819284480, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 827025920, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 827032064, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 827033088, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.4.attn_output.weight", "offset": 827254272, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.4.attn_q.bias", "offset": 828581376, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 828587520, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.4.attn_v.bias", "offset": 829914624, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 829915648, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 830136832, "shape": [ 1536 ], "size": 221184, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 830142976, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.5.ffn_gate.weight", "offset": 841432576, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 849174016, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 856915456, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 856921600, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 856922624, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.5.attn_output.weight", "offset": 857143808, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.5.attn_q.bias", "offset": 858470912, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 858477056, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.5.attn_v.bias", "offset": 859804160, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 859805184, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 860127744, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 860133888, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.6.ffn_gate.weight", "offset": 871423488, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 879164928, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 886906368, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 886912512, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 886913536, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 887134720, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.6.attn_q.bias", "offset": 888461824, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 888467968, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.6.attn_v.bias", "offset": 889795072, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 889796096, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.7.attn_norm.weight", "offset": 890118656, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 890124800, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.7.ffn_gate.weight", "offset": 901414400, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 909155840, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 916897280, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 916903424, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 916904448, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.7.attn_output.weight", "offset": 917125632, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.7.attn_q.bias", "offset": 918452736, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 918458880, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.7.attn_v.bias", "offset": 919785984, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 919787008, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 920109568, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 920115712, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.8.ffn_gate.weight", "offset": 931405312, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 939146752, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 946888192, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 946894336, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 946895360, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.8.attn_output.weight", "offset": 947116544, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.8.attn_q.bias", "offset": 948443648, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 948449792, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.8.attn_v.bias", "offset": 949776896, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 949777920, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 950100480, "shape": [ 1536 ], "size": 322560, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 950106624, "shape": [ 8960, 1536 ], "size": 6144, "type": 14 }, { "name": "blk.9.ffn_gate.weight", "offset": 961396224, "shape": [ 1536, 8960 ], "size": 11289600, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 969137664, "shape": [ 1536, 8960 ], "size": 7741440, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 976879104, "shape": [ 1536 ], "size": 7741440, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 976885248, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 976886272, "shape": [ 1536, 256 ], "size": 1024, "type": 12 }, { "name": "blk.9.attn_output.weight", "offset": 977107456, "shape": [ 1536, 1536 ], "size": 221184, "type": 12 }, { "name": "blk.9.attn_q.bias", "offset": 978434560, "shape": [ 1536 ], "size": 1327104, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 978440704, "shape": [ 1536, 1536 ], "size": 6144, "type": 12 }, { "name": "blk.9.attn_v.bias", "offset": 979767808, "shape": [ 256 ], "size": 1327104, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 979768832, "shape": [ 1536, 256 ], "size": 1024, "type": 14 }, { "name": "output_norm.weight", "offset": 980091392, "shape": [ 1536 ], "size": 322560, "type": 0 } ], "version": 3 }