phi3

Phi-3 is a family of lightweight 3B (Mini) and 14B (Medium) state-of-the-art open models by Microsoft.

3B 14B

2.1M Pulls Updated 9 hours ago

73 Tags

... /
model
d3e97425f8fb · 6.1GB
{ "metadata": { "general.architecture": "phi3", "general.file_type": 11, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 70541312, "shape": [ 5120 ], "size": 70540800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 70561792, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 109985792, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 188833792, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 188854272, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.attn_qkv.weight", "offset": 200118272, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.1.attn_norm.weight", "offset": 217014272, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 217034752, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 256458752, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 335306752, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 335327232, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.attn_qkv.weight", "offset": 346591232, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.2.attn_norm.weight", "offset": 363487232, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 363507712, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 402931712, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 481779712, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 481800192, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.attn_qkv.weight", "offset": 493064192, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.3.attn_norm.weight", "offset": 509960192, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 509980672, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 549404672, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 628252672, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 628273152, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.attn_qkv.weight", "offset": 639537152, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.4.attn_norm.weight", "offset": 656433152, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 656453632, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 695877632, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 774725632, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 774746112, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.attn_qkv.weight", "offset": 786010112, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.5.attn_norm.weight", "offset": 802906112, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 802926592, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 842350592, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 921198592, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 921219072, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.attn_qkv.weight", "offset": 932483072, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.6.ffn_up.weight", "offset": 949379072, "shape": [ 5120, 35840 ], "size": 16896000, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1028227072, "shape": [ 5120, 5120 ], "size": 78848000, "type": 11 }, { "name": "blk.6.attn_qkv.weight", "offset": 1039491072, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.10.attn_norm.weight", "offset": 1056387072, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1056407552, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 1095831552, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1174679552, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1174700032, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.attn_qkv.weight", "offset": 1185964032, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.11.attn_norm.weight", "offset": 1202860032, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1202880512, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1242304512, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1321152512, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1321172992, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.attn_qkv.weight", "offset": 1332436992, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.12.attn_norm.weight", "offset": 1349332992, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1349353472, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1388777472, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1467625472, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1467645952, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.attn_qkv.weight", "offset": 1478909952, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.13.attn_norm.weight", "offset": 1495805952, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1495826432, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1535250432, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1614098432, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1614118912, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.attn_qkv.weight", "offset": 1625382912, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.6.attn_norm.weight", "offset": 1642278912, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1642299392, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1681723392, "shape": [ 5120 ], "size": 39424000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1681743872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1681764352, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1721188352, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 1800036352, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 1800056832, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.attn_qkv.weight", "offset": 1811320832, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.8.attn_norm.weight", "offset": 1828216832, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1828237312, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1867661312, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 1946509312, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 1946529792, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.attn_qkv.weight", "offset": 1957793792, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.9.attn_norm.weight", "offset": 1974689792, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1974710272, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 2014134272, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2092982272, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2093002752, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.attn_qkv.weight", "offset": 2104266752, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.14.attn_norm.weight", "offset": 2121162752, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2121183232, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 2160607232, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2239455232, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2239475712, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.attn_qkv.weight", "offset": 2250739712, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.15.attn_norm.weight", "offset": 2267635712, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2267656192, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 2307080192, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2385928192, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2385948672, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.attn_qkv.weight", "offset": 2397212672, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.16.attn_norm.weight", "offset": 2414108672, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2414129152, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2453553152, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2532401152, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2532421632, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.attn_qkv.weight", "offset": 2543685632, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.17.attn_norm.weight", "offset": 2560581632, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2560602112, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 2600026112, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 2678874112, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 2678894592, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.attn_qkv.weight", "offset": 2690158592, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.18.attn_norm.weight", "offset": 2707054592, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2707075072, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2746499072, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 2825347072, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 2825367552, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.attn_qkv.weight", "offset": 2836631552, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.19.attn_norm.weight", "offset": 2853527552, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2853548032, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2892972032, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 2971820032, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 2971840512, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.attn_qkv.weight", "offset": 2983104512, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.20.attn_norm.weight", "offset": 3000000512, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3000020992, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 3039444992, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 3118292992, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3118313472, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.attn_qkv.weight", "offset": 3129577472, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.21.attn_output.weight", "offset": 3146473472, "shape": [ 5120, 5120 ], "size": 16896000, "type": 11 }, { "name": "blk.21.attn_qkv.weight", "offset": 3157737472, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.21.attn_norm.weight", "offset": 3174633472, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3174653952, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 3214077952, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 3292925952, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 3292946432, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3292966912, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 3332390912, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 3411238912, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 3411259392, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.attn_qkv.weight", "offset": 3422523392, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.23.attn_norm.weight", "offset": 3439419392, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3439439872, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 3478863872, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 3557711872, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 3557732352, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.attn_qkv.weight", "offset": 3568996352, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.24.attn_norm.weight", "offset": 3585892352, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3585912832, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3625336832, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 3704184832, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 3704205312, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.attn_qkv.weight", "offset": 3715469312, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.25.attn_norm.weight", "offset": 3732365312, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3732385792, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3771809792, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 3850657792, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 3850678272, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.attn_qkv.weight", "offset": 3861942272, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.26.attn_norm.weight", "offset": 3878838272, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3878858752, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3918282752, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 3997130752, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 3997151232, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.attn_qkv.weight", "offset": 4008415232, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.27.attn_norm.weight", "offset": 4025311232, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4025331712, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 4064755712, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 4143603712, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 4143624192, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.attn_qkv.weight", "offset": 4154888192, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.28.attn_output.weight", "offset": 4171784192, "shape": [ 5120, 5120 ], "size": 16896000, "type": 11 }, { "name": "blk.28.attn_qkv.weight", "offset": 4183048192, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.28.attn_norm.weight", "offset": 4199944192, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4199964672, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 4239388672, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 4318236672, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 4318257152, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4318277632, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 4357701632, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 4436549632, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 4436570112, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.attn_qkv.weight", "offset": 4447834112, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.30.attn_norm.weight", "offset": 4464730112, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 4464750592, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 4504174592, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 4583022592, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 4583043072, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.attn_qkv.weight", "offset": 4594307072, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.31.attn_norm.weight", "offset": 4611203072, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 4611223552, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 4650647552, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 4729495552, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 4729516032, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.attn_qkv.weight", "offset": 4740780032, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.32.attn_norm.weight", "offset": 4757676032, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4757696512, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 4797120512, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 4875968512, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 4875988992, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.attn_qkv.weight", "offset": 4887252992, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.33.attn_norm.weight", "offset": 4904148992, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4904169472, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 4943593472, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 5022441472, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 5022461952, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.attn_qkv.weight", "offset": 5033725952, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.34.attn_norm.weight", "offset": 5050621952, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 5050642432, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 5090066432, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 5168914432, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 5168934912, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.attn_qkv.weight", "offset": 5180198912, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.35.attn_output.weight", "offset": 5197094912, "shape": [ 5120, 5120 ], "size": 16896000, "type": 11 }, { "name": "blk.35.attn_qkv.weight", "offset": 5208358912, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "output.weight", "offset": 5225254912, "shape": [ 5120, 32064 ], "size": 16896000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 5359923712, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 5359944192, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 5399368192, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 5478216192, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 5478236672, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 5478257152, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 5517681152, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 5596529152, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 5596549632, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.attn_qkv.weight", "offset": 5607813632, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.37.attn_norm.weight", "offset": 5624709632, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 5624730112, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 5664154112, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 5743002112, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 5743022592, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.attn_qkv.weight", "offset": 5754286592, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.38.attn_norm.weight", "offset": 5771182592, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 5771203072, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 5810627072, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 5889475072, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 5889495552, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.attn_qkv.weight", "offset": 5900759552, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "blk.39.attn_norm.weight", "offset": 5917655552, "shape": [ 5120 ], "size": 16896000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5917676032, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 5957100032, "shape": [ 5120, 35840 ], "size": 39424000, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 6035948032, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 6035968512, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.attn_qkv.weight", "offset": 6047232512, "shape": [ 5120, 7680 ], "size": 11264000, "type": 11 }, { "name": "output_norm.weight", "offset": 6064128512, "shape": [ 5120 ], "size": 16896000, "type": 0 } ], "version": 3 }