phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 13, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 70541312, "shape": [ 5120 ], "size": 70540800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 70561792, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 133640192, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 212488192, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 212508672, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.attn_qkv.weight", "offset": 230531072, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 252649472, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 252669952, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 315748352, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 394596352, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 394616832, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.attn_qkv.weight", "offset": 412639232, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 434757632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 434778112, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 497856512, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 576704512, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 576724992, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.attn_qkv.weight", "offset": 594747392, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 616865792, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 616886272, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 679964672, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 758812672, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 758833152, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.attn_qkv.weight", "offset": 776855552, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 798973952, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 798994432, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 862072832, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 940920832, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 940941312, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.attn_qkv.weight", "offset": 958963712, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 981082112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 981102592, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1044180992, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1123028992, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1123049472, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.attn_qkv.weight", "offset": 1141071872, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1163190272, "shape": [ 5120, 35840 ], "size": 22118400, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1242038272, "shape": [ 5120, 5120 ], "size": 78848000, "type": 13 }, { "name": "blk.6.attn_qkv.weight", "offset": 1260060672, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1282179072, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1282199552, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 1345277952, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1424125952, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1424146432, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.attn_qkv.weight", "offset": 1442168832, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1464287232, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1464307712, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 1527386112, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1606234112, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1606254592, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.11.attn_qkv.weight", "offset": 1624276992, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1646395392, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1646415872, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 1709494272, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1788342272, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1788362752, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.attn_qkv.weight", "offset": 1806385152, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1828503552, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1828524032, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 1891602432, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1970450432, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1970470912, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.attn_qkv.weight", "offset": 1988493312, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 2010611712, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2010632192, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 2073710592, "shape": [ 5120 ], "size": 63078400, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2073731072, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2073751552, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 2136829952, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 2215677952, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2215698432, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.attn_qkv.weight", "offset": 2233720832, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2255839232, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2255859712, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 2318938112, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 2397786112, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2397806592, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.attn_qkv.weight", "offset": 2415828992, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2437947392, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2437967872, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 2501046272, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2579894272, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2579914752, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.9.attn_qkv.weight", "offset": 2597937152, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2620055552, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2620076032, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 2683154432, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2762002432, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2762022912, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.attn_qkv.weight", "offset": 2780045312, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 2802163712, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2802184192, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 2865262592, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2944110592, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2944131072, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.attn_qkv.weight", "offset": 2962153472, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2984271872, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2984292352, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 3047370752, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 3126218752, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3126239232, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.16.attn_qkv.weight", "offset": 3144261632, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 3166380032, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3166400512, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 3229478912, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 3308326912, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3308347392, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.attn_qkv.weight", "offset": 3326369792, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3348488192, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3348508672, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 3411587072, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 3490435072, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3490455552, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.attn_qkv.weight", "offset": 3508477952, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 3530596352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3530616832, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 3593695232, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "output.weight\u0002\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000", "offset": 3672543232, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3672563712, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.19.attn_qkv.weight", "offset": 3690586112, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3712704512, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3712724992, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 3775803392, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 3854651392, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3854671872, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.attn_qkv.weight", "offset": 3872694272, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 3894812672, "shape": [ 5120, 5120 ], "size": 22118400, "type": 13 }, { "name": "blk.21.attn_qkv.weight", "offset": 3912835072, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 3934953472, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3934973952, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 3998052352, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 4076900352, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4076920832, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4076941312, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 4140019712, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 4218867712, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4218888192, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.22.attn_qkv.weight", "offset": 4236910592, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 4259028992, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4259049472, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 4322127872, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 4400975872, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4400996352, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.attn_qkv.weight", "offset": 4419018752, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4441137152, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4441157632, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 4504236032, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 4583084032, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4583104512, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.attn_qkv.weight", "offset": 4601126912, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4623245312, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4623265792, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 4686344192, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 4765192192, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 4765212672, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.25.attn_qkv.weight", "offset": 4783235072, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 4805353472, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4805373952, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 4868452352, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 4947300352, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 4947320832, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.attn_qkv.weight", "offset": 4965343232, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 4987461632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4987482112, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 5050560512, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 5129408512, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5129428992, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.attn_qkv.weight", "offset": 5147451392, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 5169569792, "shape": [ 5120, 5120 ], "size": 22118400, "type": 13 }, { "name": "blk.28.attn_qkv.weight", "offset": 5187592192, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 5209710592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5209731072, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 5272809472, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 5351657472, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 5351677952, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5351698432, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 5414776832, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 5493624832, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5493645312, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.attn_qkv.weight", "offset": 5511667712, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 5533786112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5533806592, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 5596884992, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 5675732992, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 5675753472, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.attn_qkv.weight", "offset": 5693775872, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 5715894272, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5715914752, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 5778993152, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 5857841152, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 5857861632, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.31.attn_qkv.weight", "offset": 5875884032, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 5898002432, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 5898022912, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 5961101312, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 6039949312, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6039969792, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.attn_qkv.weight", "offset": 6057992192, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 6080110592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6080131072, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 6143209472, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 6222057472, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 6222077952, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.attn_qkv.weight", "offset": 6240100352, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 6262218752, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6262239232, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.34.ffn_up.weight", "offset": 6325317632, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 6404165632, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 6404186112, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.34.attn_qkv.weight", "offset": 6422208512, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 6444326912, "shape": [ 5120, 5120 ], "size": 22118400, "type": 13 }, { "name": "blk.35.attn_qkv.weight", "offset": 6462349312, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "output.weight", "offset": 6484467712, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 6619136512, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6619156992, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 6682235392, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 6761083392, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 6761103872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6761124352, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 6824202752, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 6903050752, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 6903071232, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.36.attn_qkv.weight", "offset": 6921093632, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 6943212032, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 6943232512, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 7006310912, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 7085158912, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 7085179392, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.37.attn_qkv.weight", "offset": 7103201792, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 7125320192, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7125340672, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 7188419072, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 7267267072, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 7267287552, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.38.attn_qkv.weight", "offset": 7285309952, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 7307428352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7307448832, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 7370527232, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 7449375232, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 7449395712, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.39.attn_qkv.weight", "offset": 7467418112, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "output_norm.weight", "offset": 7489536512, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }