qwen2

{ "metadata": { "general.architecture": "qwen2", "general.file_type": 16, "general.name": "Qwen2-1.5B-Instruct", "general.quantization_version": 2, "qwen2.attention.head_count": 12, "qwen2.attention.head_count_kv": 2, "qwen2.attention.layer_norm_rms_epsilon": 0.000001, "qwen2.block_count": 28, "qwen2.context_length": 32768, "qwen2.embedding_length": 1536, "qwen2.feed_forward_length": 8960, "qwen2.rope.freq_base": 1000000, "tokenizer.ggml.bos_token_id": 151643, "tokenizer.ggml.eos_token_id": 151645, "tokenizer.ggml.merges": "... (151387 values)", "tokenizer.ggml.model": "gpt2", "tokenizer.ggml.padding_token_id": 151643, "tokenizer.ggml.pre": "qwen2", "tokenizer.ggml.token_type": "... (151936 values)", "tokenizer.ggml.tokens": "... (151936 values)" }, "num_params": 1543714304, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 191439360, "shape": [ 1536 ], "size": 191439360, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 191445504, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.0.ffn_gate.weight", "offset": 200907264, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 210369024, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 219830784, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.0.attn_k.bias", "offset": 219836928, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.0.attn_k.weight", "offset": 219837952, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.0.attn_output.weight", "offset": 220108288, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.0.attn_q.bias", "offset": 221730304, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.0.attn_q.weight", "offset": 221736448, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.0.attn_v.bias", "offset": 223358464, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.0.attn_v.weight", "offset": 223359488, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 223629824, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 223635968, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.1.ffn_gate.weight", "offset": 233097728, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 242559488, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 252021248, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.1.attn_k.bias", "offset": 252027392, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.1.attn_k.weight", "offset": 252028416, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.1.attn_output.weight", "offset": 252298752, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.1.attn_q.bias", "offset": 253920768, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.1.attn_q.weight", "offset": 253926912, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.1.attn_v.bias", "offset": 255548928, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.1.attn_v.weight", "offset": 255549952, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 255820288, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 255826432, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.10.ffn_gate.weight", "offset": 265288192, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 274749952, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 284211712, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.10.attn_k.bias", "offset": 284217856, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.10.attn_k.weight", "offset": 284218880, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.10.attn_output.weight", "offset": 284489216, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.10.attn_q.bias", "offset": 286111232, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.10.attn_q.weight", "offset": 286117376, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.10.attn_v.bias", "offset": 287739392, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.10.attn_v.weight", "offset": 287740416, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 288010752, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 288016896, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "\u0000\u0000\u0000blk.11.attn_k.weigh", "offset": 297478656, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 306940416, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 316402176, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.11.attn_k.bias", "offset": 316408320, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.11.attn_k.weight", "offset": 316409344, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.11.attn_output.weight", "offset": 316679680, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.11.attn_q.bias", "offset": 318301696, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.11.attn_q.weight", "offset": 318307840, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.11.attn_v.bias", "offset": 319929856, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.11.attn_v.weight", "offset": 319930880, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 320201216, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 320207360, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.12.ffn_gate.weight", "offset": 329669120, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 339130880, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 348592640, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.12.attn_k.bias", "offset": 348598784, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.12.attn_k.weight", "offset": 348599808, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.12.attn_output.weight", "offset": 348870144, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.12.attn_q.bias", "offset": 350492160, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.12.attn_q.weight", "offset": 350498304, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.12.attn_v.bias", "offset": 352120320, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.12.attn_v.weight", "offset": 352121344, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 352391680, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 352397824, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.13.ffn_gate.weight", "offset": 361859584, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 371321344, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 380783104, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.13.attn_k.bias", "offset": 380789248, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.13.attn_k.weight", "offset": 380790272, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.13.attn_output.weight", "offset": 381060608, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.13.attn_q.bias", "offset": 382682624, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.13.attn_q.weight", "offset": 382688768, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.13.attn_v.bias", "offset": 384310784, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.13.attn_v.weight", "offset": 384311808, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 384582144, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 384588288, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.14.ffn_gate.weight", "offset": 394050048, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 403511808, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 412973568, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.14.attn_k.bias", "offset": 412979712, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.14.attn_k.weight", "offset": 412980736, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.14.attn_output.weight", "offset": 413251072, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.14.attn_q.bias", "offset": 414873088, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.14.attn_q.weight", "offset": 414879232, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.14.attn_v.bias", "offset": 416501248, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.14.attn_v.weight", "offset": 416502272, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 416772608, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 416778752, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.15.ffn_gate.weight", "offset": 426240512, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 435702272, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 445164032, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.15.attn_k.bias", "offset": 445170176, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.15.attn_k.weight", "offset": 445171200, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.15.attn_output.weight", "offset": 445441536, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.15.attn_q.bias", "offset": 447063552, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.15.attn_q.weight", "offset": 447069696, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.15.attn_v.bias", "offset": 448691712, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.15.attn_v.weight", "offset": 448692736, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 448963072, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 448969216, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.16.ffn_gate.weight", "offset": 458430976, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 467892736, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 477354496, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.16.attn_k.bias", "offset": 477360640, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.16.attn_k.weight", "offset": 477361664, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.16.attn_output.weight", "offset": 477632000, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.16.attn_q.bias", "offset": 479254016, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.16.attn_q.weight", "offset": 479260160, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.16.attn_v.bias", "offset": 480882176, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.16.attn_v.weight", "offset": 480883200, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 481153536, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 481159680, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.17.ffn_gate.weight", "offset": 490621440, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 500083200, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 509544960, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.17.attn_k.bias", "offset": 509551104, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.17.attn_k.weight", "offset": 509552128, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.17.attn_output.weight", "offset": 509822464, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.17.attn_q.bias", "offset": 511444480, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.17.attn_q.weight", "offset": 511450624, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.17.attn_v.bias", "offset": 513072640, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.17.attn_v.weight", "offset": 513073664, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 513344000, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 513350144, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.18.ffn_gate.weight", "offset": 522811904, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 532273664, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 541735424, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.18.attn_k.bias", "offset": 541741568, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.18.attn_k.weight", "offset": 541742592, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.18.attn_output.weight", "offset": 542012928, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.18.attn_q.bias", "offset": 543634944, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.18.attn_q.weight", "offset": 543641088, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.18.attn_v.bias", "offset": 545263104, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.18.attn_v.weight", "offset": 545264128, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 545534464, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 545540608, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.19.ffn_gate.weight", "offset": 555002368, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 564464128, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 573925888, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.19.attn_k.bias", "offset": 573932032, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.19.attn_k.weight", "offset": 573933056, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.19.attn_output.weight", "offset": 574203392, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.19.attn_q.bias", "offset": 575825408, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.19.attn_q.weight", "offset": 575831552, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.19.attn_v.bias", "offset": 577453568, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.19.attn_v.weight", "offset": 577454592, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 577724928, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 577731072, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.2.ffn_gate.weight", "offset": 587192832, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 596654592, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 606116352, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.2.attn_k.bias", "offset": 606122496, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.2.attn_k.weight", "offset": 606123520, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.2.attn_output.weight", "offset": 606393856, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.2.attn_q.bias", "offset": 608015872, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.2.attn_q.weight", "offset": 608022016, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.2.attn_v.bias", "offset": 609644032, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.2.attn_v.weight", "offset": 609645056, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 609915392, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 609921536, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.20.ffn_gate.weight", "offset": 619383296, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 628845056, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 638306816, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.20.attn_k.bias", "offset": 638312960, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.20.attn_k.weight", "offset": 638313984, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.20.attn_output.weight", "offset": 638584320, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.20.attn_q.bias", "offset": 640206336, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.20.attn_q.weight", "offset": 640212480, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.20.attn_v.bias", "offset": 641834496, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.20.attn_v.weight", "offset": 641835520, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 642105856, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 642112000, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.21.ffn_gate.weight", "offset": 651573760, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 661035520, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 670497280, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.21.attn_k.bias", "offset": 670503424, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.21.attn_k.weight", "offset": 670504448, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 670774784, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.21.attn_q.bias", "offset": 672396800, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.21.attn_q.weight", "offset": 672402944, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.21.attn_v.bias", "offset": 674024960, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.21.attn_v.weight", "offset": 674025984, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 674296320, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 674302464, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.22.ffn_gate.weight", "offset": 683764224, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 693225984, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 702687744, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.22.attn_k.bias", "offset": 702693888, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.22.attn_k.weight", "offset": 702694912, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.22.attn_output.weight", "offset": 702965248, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.22.attn_q.bias", "offset": 704587264, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.22.attn_q.weight", "offset": 704593408, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.22.attn_v.bias", "offset": 706215424, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.22.attn_v.weight", "offset": 706216448, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 706486784, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 706492928, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.23.ffn_gate.weight", "offset": 715954688, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 725416448, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 734878208, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.23.attn_k.bias", "offset": 734884352, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.23.attn_k.weight", "offset": 734885376, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.23.attn_output.weight", "offset": 735155712, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.23.attn_q.bias", "offset": 736777728, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.23.attn_q.weight", "offset": 736783872, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.23.attn_v.bias", "offset": 738405888, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.23.attn_v.weight", "offset": 738406912, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 738677248, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 738683392, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.24.ffn_gate.weight", "offset": 748145152, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 757606912, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 767068672, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.24.attn_k.bias", "offset": 767074816, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.24.attn_k.weight", "offset": 767075840, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.24.attn_output.weight", "offset": 767346176, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.24.attn_q.bias", "offset": 768968192, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.24.attn_q.weight", "offset": 768974336, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.24.attn_v.bias", "offset": 770596352, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.24.attn_v.weight", "offset": 770597376, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 770867712, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 770873856, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.25.ffn_gate.weight", "offset": 780335616, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 789797376, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 799259136, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.25.attn_k.bias", "offset": 799265280, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.25.attn_k.weight", "offset": 799266304, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.25.attn_output.weight", "offset": 799536640, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.25.attn_q.bias", "offset": 801158656, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.25.attn_q.weight", "offset": 801164800, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.25.attn_v.bias", "offset": 802786816, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.25.attn_v.weight", "offset": 802787840, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 803058176, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 803064320, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.26.ffn_gate.weight", "offset": 812526080, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 821987840, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 831449600, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.26.attn_k.bias", "offset": 831455744, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.26.attn_k.weight", "offset": 831456768, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.26.attn_output.weight", "offset": 831727104, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.26.attn_q.bias", "offset": 833349120, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.26.attn_q.weight", "offset": 833355264, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.26.attn_v.bias", "offset": 834977280, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.26.attn_v.weight", "offset": 834978304, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 835248640, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 835254784, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.27.ffn_gate.weight", "offset": 844716544, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 854178304, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 863640064, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.27.attn_k.bias", "offset": 863646208, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.27.attn_k.weight", "offset": 863647232, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.27.attn_output.weight", "offset": 863917568, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.27.attn_q.bias", "offset": 865539584, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.27.attn_q.weight", "offset": 865545728, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.27.attn_v.bias", "offset": 867167744, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.27.attn_v.weight", "offset": 867168768, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 867439104, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 867445248, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.3.ffn_gate.weight", "offset": 876907008, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 886368768, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 895830528, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.3.attn_k.bias", "offset": 895836672, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.3.attn_k.weight", "offset": 895837696, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.3.attn_output.weight", "offset": 896108032, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.3.attn_q.bias", "offset": 897730048, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.3.attn_q.weight", "offset": 897736192, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.3.attn_v.bias", "offset": 899358208, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.3.attn_v.weight", "offset": 899359232, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 899629568, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 899635712, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.4.ffn_gate.weight", "offset": 909097472, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 918559232, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 928020992, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.4.attn_k.bias", "offset": 928027136, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.4.attn_k.weight", "offset": 928028160, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.4.attn_output.weight", "offset": 928298496, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.4.attn_q.bias", "offset": 929920512, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.4.attn_q.weight", "offset": 929926656, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.4.attn_v.bias", "offset": 931548672, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.4.attn_v.weight", "offset": 931549696, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 931820032, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 931826176, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.5.ffn_gate.weight", "offset": 941287936, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 950749696, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 960211456, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.5.attn_k.bias", "offset": 960217600, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.5.attn_k.weight", "offset": 960218624, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.5.attn_output.weight", "offset": 960488960, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.5.attn_q.bias", "offset": 962110976, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.5.attn_q.weight", "offset": 962117120, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.5.attn_v.bias", "offset": 963739136, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.5.attn_v.weight", "offset": 963740160, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 964010496, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 964016640, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.6.ffn_gate.weight", "offset": 973478400, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 982940160, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 992401920, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.6.attn_k.bias", "offset": 992408064, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.6.attn_k.weight", "offset": 992409088, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 992679424, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.6.attn_q.bias", "offset": 994301440, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.6.attn_q.weight", "offset": 994307584, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.6.attn_v.bias", "offset": 995929600, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.6.attn_v.weight", "offset": 995930624, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 996200960, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 996207104, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.7.ffn_gate.weight", "offset": 1005668864, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 1015130624, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 1024592384, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.7.attn_k.bias", "offset": 1024598528, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.7.attn_k.weight", "offset": 1024599552, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.7.attn_output.weight", "offset": 1024869888, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.7.attn_q.bias", "offset": 1026491904, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.7.attn_q.weight", "offset": 1026498048, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.7.attn_v.bias", "offset": 1028120064, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.7.attn_v.weight", "offset": 1028121088, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 1028391424, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1028397568, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.8.ffn_gate.weight", "offset": 1037859328, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 1047321088, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 1056782848, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.8.attn_k.bias", "offset": 1056788992, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.8.attn_k.weight", "offset": 1056790016, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.8.attn_output.weight", "offset": 1057060352, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.8.attn_q.bias", "offset": 1058682368, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.8.attn_q.weight", "offset": 1058688512, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.8.attn_v.bias", "offset": 1060310528, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.8.attn_v.weight", "offset": 1060311552, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1060581888, "shape": [ 1536 ], "size": 270336, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1060588032, "shape": [ 8960, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.9.ffn_gate.weight", "offset": 1070049792, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 1079511552, "shape": [ 1536, 8960 ], "size": 9461760, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 1088973312, "shape": [ 1536 ], "size": 9461760, "type": 0 }, { "name": "blk.9.attn_k.bias", "offset": 1088979456, "shape": [ 256 ], "size": 6144, "type": 0 }, { "name": "blk.9.attn_k.weight", "offset": 1088980480, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "blk.9.attn_output.weight", "offset": 1089250816, "shape": [ 1536, 1536 ], "size": 270336, "type": 13 }, { "name": "blk.9.attn_q.bias", "offset": 1090872832, "shape": [ 1536 ], "size": 1622016, "type": 0 }, { "name": "blk.9.attn_q.weight", "offset": 1090878976, "shape": [ 1536, 1536 ], "size": 6144, "type": 13 }, { "name": "blk.9.attn_v.bias", "offset": 1092500992, "shape": [ 256 ], "size": 1622016, "type": 0 }, { "name": "blk.9.attn_v.weight", "offset": 1092502016, "shape": [ 1536, 256 ], "size": 1024, "type": 13 }, { "name": "output_norm.weight", "offset": 1092772352, "shape": [ 1536 ], "size": 270336, "type": 0 } ], "version": 3 }