phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 15, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 32, "phi3.attention.head_count_kv": 32, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 32, "phi3.context_length": 4096, "phi3.embedding_length": 3072, "phi3.feed_forward_length": 8192, "phi3.rope.dimension_count": 96, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 3821079552, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 55406592, "shape": [ 3072 ], "size": 55406592, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 55418880, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 76062720, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 104374272, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 104386560, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.0.attn_qkv.weight", "offset": 109694976, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 129159168, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 129171456, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 149815296, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 178126848, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 178139136, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.1.attn_qkv.weight", "offset": 183447552, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 202911744, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 202924032, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 223567872, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 251879424, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 251891712, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.10.attn_qkv.weight", "offset": 257200128, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 276664320, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 276676608, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 297320448, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 325632000, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 325644288, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.11.attn_qkv.weight", "offset": 330952704, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 350416896, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 350429184, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 364584960, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 392896512, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 392908800, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.12.attn_qkv.weight", "offset": 398217216, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 417681408, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 417693696, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 431849472, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 460161024, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 460173312, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.13.attn_qkv.weight", "offset": 465481728, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 484945920, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 484958208, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 505602048, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 533913600, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 533925888, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.14.attn_qkv.weight", "offset": 539234304, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 558698496, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 558710784, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 572866560, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 601178112, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 601190400, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.15.attn_qkv.weight", "offset": 606498816, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 625963008, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 625975296, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 640131072, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 668442624, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 668454912, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.16.attn_qkv.weight", "offset": 673763328, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 693227520, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 693239808, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 713883648, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 742195200, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 742207488, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.17.attn_qkv.weight", "offset": 747515904, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 766980096, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 766992384, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 781148160, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 809459712, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 809472000, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.18.attn_qkv.weight", "offset": 814780416, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 834244608, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 834256896, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 848412672, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 876724224, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 876736512, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.19.attn_qkv.weight", "offset": 882044928, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 901509120, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 901521408, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 922165248, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 950476800, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 950489088, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.2.attn_qkv.weight", "offset": 955797504, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 975261696, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 975273984, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 989429760, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 1017741312, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 1017753600, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.20.attn_qkv.weight", "offset": 1023062016, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 1042526208, "shape": [ 3072, 3072 ], "size": 19464192, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 1047834624, "shape": [ 3072 ], "size": 5308416, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1047846912, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 1062002688, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 1090314240, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1090326528, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.3.attn_qkv.weight", "offset": 1095634944, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 1115099136, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1115111424, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1135755264, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1164066816, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1164079104, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.4.attn_qkv.weight", "offset": 1169387520, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1188851712, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1188864000, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1203019776, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1231331328, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1231343616, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.5.attn_qkv.weight", "offset": 1236652032, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 1256116224, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1256128512, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1270284288, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 1298595840, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.6.attn_output.weight", "offset": 1298608128, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.6.attn_qkv.weight", "offset": 1303916544, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.7.attn_norm.weight", "offset": 1323380736, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1323393024, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 1344036864, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 1372348416, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 1372360704, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.7.attn_qkv.weight", "offset": 1377669120, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 1397133312, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1397145600, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 1411301376, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 1439612928, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 1439625216, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.8.attn_qkv.weight", "offset": 1444933632, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 1464397824, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1464410112, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 1478565888, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 1506877440, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 1506889728, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.9.attn_qkv.weight", "offset": 1512198144, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "output.weight", "offset": 1531662336, "shape": [ 3072, 32064 ], "size": 19464192, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 1612463616, "shape": [ 3072 ], "size": 80801280, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1612475904, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 1633119744, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 1661431296, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.21.attn_qkv.weight", "offset": 1661443584, "shape": [ 3072, 9216 ], "size": 12288, "type": 13 }, { "name": "blk.22.attn_norm.weight", "offset": 1680907776, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 1680920064, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 1695075840, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 1723387392, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 1723399680, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.22.attn_qkv.weight", "offset": 1728708096, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 1748172288, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 1748184576, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 1762340352, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 1790651904, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 1790664192, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.23.attn_qkv.weight", "offset": 1795972608, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 1815436800, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 1815449088, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 1836092928, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 1864404480, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 1864416768, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.24.attn_qkv.weight", "offset": 1869725184, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 1889189376, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 1889201664, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 1903357440, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 1931668992, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 1931681280, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.25.attn_qkv.weight", "offset": 1936989696, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 1956453888, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 1956466176, "shape": [ 8192, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 1970621952, "shape": [ 3072, 16384 ], "size": 14155776, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 1998933504, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 1998945792, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.26.attn_qkv.weight", "offset": 2004254208, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 2023718400, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 2023730688, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 2044374528, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 2072686080, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 2072698368, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.27.attn_qkv.weight", "offset": 2078006784, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 2097470976, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 2097483264, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 2118127104, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 2146438656, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.28.attn_output.weight", "offset": 2146450944, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.28.attn_qkv.weight", "offset": 2151759360, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.29.attn_norm.weight", "offset": 2171223552, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 2171235840, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 2191879680, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 2220191232, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 2220203520, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.29.attn_qkv.weight", "offset": 2225511936, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 2244976128, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 2244988416, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 2265632256, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 2293943808, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 2293956096, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.30.attn_qkv.weight", "offset": 2299264512, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 2318728704, "shape": [ 3072 ], "size": 19464192, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 2318740992, "shape": [ 8192, 3072 ], "size": 12288, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 2339384832, "shape": [ 3072, 16384 ], "size": 20643840, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 2367696384, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 2367708672, "shape": [ 3072, 3072 ], "size": 12288, "type": 12 }, { "name": "blk.31.attn_qkv.weight", "offset": 2373017088, "shape": [ 3072, 9216 ], "size": 5308416, "type": 13 }, { "name": "output_norm.weight", "offset": 2392481280, "shape": [ 3072 ], "size": 19464192, "type": 0 } ], "version": 3 }