phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 10, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 53867520, "shape": [ 5120 ], "size": 53867520, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 53888000, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 93312000, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 153523200, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 153543680, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.attn_qkv.weight", "offset": 164807680, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.1.attn_norm.weight", "offset": 177710080, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 177730560, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 217154560, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 277365760, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 277386240, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.attn_qkv.weight", "offset": 288650240, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.2.attn_norm.weight", "offset": 301552640, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 301573120, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 340997120, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 401208320, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 401228800, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.attn_qkv.weight", "offset": 412492800, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.3.attn_norm.weight", "offset": 425395200, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 425415680, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 464839680, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 525050880, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 525071360, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.attn_qkv.weight", "offset": 536335360, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.4.attn_norm.weight", "offset": 549237760, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 549258240, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 588682240, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 648893440, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 648913920, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.attn_qkv.weight", "offset": 660177920, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.5.attn_norm.weight", "offset": 673080320, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 673100800, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 712524800, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 772736000, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 772756480, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.attn_qkv.weight", "offset": 784020480, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.6.ffn_up.weight", "offset": 796922880, "shape": [ 5120, 35840 ], "size": 12902400, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 857134080, "shape": [ 5120, 5120 ], "size": 60211200, "type": 11 }, { "name": "blk.6.attn_qkv.weight", "offset": 868398080, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.10.attn_norm.weight", "offset": 881300480, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 881320960, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 920744960, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 980956160, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 980976640, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.attn_qkv.weight", "offset": 992240640, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.11.attn_norm.weight", "offset": 1005143040, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1005163520, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1044587520, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 1104798720, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1104819200, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.attn_qkv.weight", "offset": 1116083200, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.12.attn_norm.weight", "offset": 1128985600, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1129006080, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1168430080, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 1228641280, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1228661760, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.attn_qkv.weight", "offset": 1239925760, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.13.attn_norm.weight", "offset": 1252828160, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1252848640, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1292272640, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 1352483840, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1352504320, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.attn_qkv.weight", "offset": 1363768320, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.6.attn_norm.weight", "offset": 1376670720, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1376691200, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1416115200, "shape": [ 5120 ], "size": 39424000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1416135680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1416156160, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1455580160, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 1515791360, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 1515811840, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.attn_qkv.weight", "offset": 1527075840, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.8.attn_norm.weight", "offset": 1539978240, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1539998720, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1579422720, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 1639633920, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 1639654400, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.attn_qkv.weight", "offset": 1650918400, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.9.attn_norm.weight", "offset": 1663820800, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1663841280, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1703265280, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 1763476480, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 1763496960, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.attn_qkv.weight", "offset": 1774760960, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.14.attn_norm.weight", "offset": 1787663360, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1787683840, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1827107840, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 1887319040, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1887339520, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.attn_qkv.weight", "offset": 1898603520, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.15.attn_norm.weight", "offset": 1911505920, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1911526400, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1950950400, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 2011161600, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2011182080, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.attn_qkv.weight", "offset": 2022446080, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.16.attn_norm.weight", "offset": 2035348480, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2035368960, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2074792960, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 2135004160, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2135024640, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.attn_qkv.weight", "offset": 2146288640, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.17.attn_norm.weight", "offset": 2159191040, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2159211520, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 2198635520, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 2258846720, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 2258867200, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.attn_qkv.weight", "offset": 2270131200, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.18.attn_norm.weight", "offset": 2283033600, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2283054080, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2322478080, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 2382689280, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 2382709760, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.attn_qkv.weight", "offset": 2393973760, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.19.attn_norm.weight", "offset": 2406876160, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2406896640, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2446320640, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 2506531840, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 2506552320, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.attn_qkv.weight", "offset": 2517816320, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.20.attn_norm.weight", "offset": 2530718720, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2530739200, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2570163200, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 2630374400, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 2630394880, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.attn_qkv.weight", "offset": 2641658880, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 2654561280, "shape": [ 5120, 5120 ], "size": 12902400, "type": 11 }, { "name": "blk.21.attn_qkv.weight", "offset": 2665825280, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.21.attn_norm.weight", "offset": 2678727680, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2678748160, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2718172160, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 2778383360, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 2778403840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2778424320, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2817848320, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 2878059520, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 2878080000, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.attn_qkv.weight", "offset": 2889344000, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.23.attn_norm.weight", "offset": 2902246400, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2902266880, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2941690880, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 3001902080, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 3001922560, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.attn_qkv.weight", "offset": 3013186560, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.24.attn_norm.weight", "offset": 3026088960, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3026109440, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3065533440, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 3125744640, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 3125765120, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.attn_qkv.weight", "offset": 3137029120, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.25.attn_norm.weight", "offset": 3149931520, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3149952000, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3189376000, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 3249587200, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 3249607680, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.attn_qkv.weight", "offset": 3260871680, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.26.attn_norm.weight", "offset": 3273774080, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3273794560, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3313218560, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 3373429760, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 3373450240, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.attn_qkv.weight", "offset": 3384714240, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.27.attn_norm.weight", "offset": 3397616640, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3397637120, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 3437061120, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.27.ffn_norm.weight", "offset": 3497272320, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3497292800, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.attn_qkv.weight", "offset": 3508556800, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 3521459200, "shape": [ 5120, 5120 ], "size": 12902400, "type": 11 }, { "name": "blk.28.attn_qkv.weight", "offset": 3532723200, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.28.attn_norm.weight", "offset": 3545625600, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3545646080, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 3585070080, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.28.ffn_norm.weight", "offset": 3645281280, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 3645301760, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3645322240, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 3684746240, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.29.ffn_norm.weight", "offset": 3744957440, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 3744977920, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.attn_qkv.weight", "offset": 3756241920, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.30.attn_norm.weight", "offset": 3769144320, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3769164800, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 3808588800, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.30.ffn_norm.weight", "offset": 3868800000, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 3868820480, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.attn_qkv.weight", "offset": 3880084480, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.31.attn_norm.weight", "offset": 3892986880, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 3893007360, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 3932431360, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.31.ffn_norm.weight", "offset": 3992642560, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3992663040, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.attn_qkv.weight", "offset": 4003927040, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.32.attn_norm.weight", "offset": 4016829440, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4016849920, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 4056273920, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.32.ffn_norm.weight", "offset": 4116485120, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 4116505600, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.attn_qkv.weight", "offset": 4127769600, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.33.attn_norm.weight", "offset": 4140672000, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4140692480, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 4180116480, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.33.ffn_norm.weight", "offset": 4240327680, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 4240348160, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.attn_qkv.weight", "offset": 4251612160, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.34.attn_norm.weight", "offset": 4264514560, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4264535040, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 4303959040, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.34.ffn_norm.weight", "offset": 4364170240, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 4364190720, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.attn_qkv.weight", "offset": 4375454720, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 4388357120, "shape": [ 5120, 5120 ], "size": 12902400, "type": 11 }, { "name": "blk.35.attn_qkv.weight", "offset": 4399621120, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "output.weight", "offset": 4412523520, "shape": [ 5120, 32064 ], "size": 12902400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 4547192320, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4547212800, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 4586636800, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.35.ffn_norm.weight", "offset": 4646848000, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 4646868480, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4646888960, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 4686312960, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.36.ffn_norm.weight", "offset": 4746524160, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 4746544640, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.attn_qkv.weight", "offset": 4757808640, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.37.attn_norm.weight", "offset": 4770711040, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4770731520, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 4810155520, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.37.ffn_norm.weight", "offset": 4870366720, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 4870387200, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.attn_qkv.weight", "offset": 4881651200, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.38.attn_norm.weight", "offset": 4894553600, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4894574080, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 4933998080, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.38.ffn_norm.weight", "offset": 4994209280, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 4994229760, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.attn_qkv.weight", "offset": 5005493760, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.39.attn_norm.weight", "offset": 5018396160, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5018416640, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 5057840640, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.39.ffn_norm.weight", "offset": 5118051840, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 5118072320, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.attn_qkv.weight", "offset": 5129336320, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "output_norm.weight", "offset": 5142238720, "shape": [ 5120 ], "size": 12902400, "type": 0 } ], "version": 3 }