phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 10, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 10 }, { "name": "blk.0.attn_norm.weight", "offset": 53868032, "shape": [ 5120 ], "size": 53867520, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 53888512, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.ffn_up.weight", "offset": 93312512, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.0.ffn_norm.weight", "offset": 153523712, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 153544192, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.0.attn_qkv.weight", "offset": 164808192, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.1.attn_norm.weight", "offset": 177710592, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 177731072, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.ffn_up.weight", "offset": 217155072, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.1.ffn_norm.weight", "offset": 277366272, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 277386752, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.1.attn_qkv.weight", "offset": 288650752, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.2.attn_norm.weight", "offset": 301553152, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 301573632, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.ffn_up.weight", "offset": 340997632, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.2.ffn_norm.weight", "offset": 401208832, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 401229312, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.2.attn_qkv.weight", "offset": 412493312, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.3.attn_norm.weight", "offset": 425395712, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 425416192, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.ffn_up.weight", "offset": 464840192, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.3.ffn_norm.weight", "offset": 525051392, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 525071872, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.3.attn_qkv.weight", "offset": 536335872, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.4.attn_norm.weight", "offset": 549238272, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 549258752, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.ffn_up.weight", "offset": 588682752, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.4.ffn_norm.weight", "offset": 648893952, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 648914432, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.4.attn_qkv.weight", "offset": 660178432, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.5.attn_norm.weight", "offset": 673080832, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 673101312, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.ffn_up.weight", "offset": 712525312, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.5.ffn_norm.weight", "offset": 772736512, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 772756992, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.5.attn_qkv.weight", "offset": 784020992, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.6.ffn_up.weight", "offset": 796923392, "shape": [ 5120, 35840 ], "size": 12902400, "type": 10 }, { "name": "blk.6.attn_output.weight", "offset": 857134592, "shape": [ 5120, 5120 ], "size": 60211200, "type": 11 }, { "name": "blk.6.attn_qkv.weight", "offset": 868398592, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.10.attn_norm.weight", "offset": 881300992, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 881321472, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.ffn_up.weight", "offset": 920745472, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.10.ffn_norm.weight", "offset": 980956672, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 980977152, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.10.attn_qkv.weight", "offset": 992241152, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.11.attn_norm.weight", "offset": 1005143552, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1005164032, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.ffn_up.weight", "offset": 1044588032, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.11.ffn_norm.weight", "offset": 1104799232, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1104819712, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.11.attn_qkv.weight", "offset": 1116083712, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.12.attn_norm.weight", "offset": 1128986112, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1129006592, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.ffn_up.weight", "offset": 1168430592, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.12.ffn_norm.weight", "offset": 1228641792, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1228662272, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.12.attn_qkv.weight", "offset": 1239926272, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.13.attn_norm.weight", "offset": 1252828672, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1252849152, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.ffn_up.weight", "offset": 1292273152, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.13.ffn_norm.weight", "offset": 1352484352, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1352504832, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.13.attn_qkv.weight", "offset": 1363768832, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.6.attn_norm.weight", "offset": 1376671232, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1376691712, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.6.ffn_norm.weight", "offset": 1416115712, "shape": [ 5120 ], "size": 39424000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1416136192, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1416156672, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.ffn_up.weight", "offset": 1455580672, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.7.ffn_norm.weight", "offset": 1515791872, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 1515812352, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.7.attn_qkv.weight", "offset": 1527076352, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.8.attn_norm.weight", "offset": 1539978752, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1539999232, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.ffn_up.weight", "offset": 1579423232, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.8.ffn_norm.weight", "offset": 1639634432, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 1639654912, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.8.attn_qkv.weight", "offset": 1650918912, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.9.attn_norm.weight", "offset": 1663821312, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1663841792, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.ffn_up.weight", "offset": 1703265792, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.9.ffn_norm.weight", "offset": 1763476992, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 1763497472, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.9.attn_qkv.weight", "offset": 1774761472, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.14.attn_norm.weight", "offset": 1787663872, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1787684352, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.ffn_up.weight", "offset": 1827108352, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.14.ffn_norm.weight", "offset": 1887319552, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1887340032, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.14.attn_qkv.weight", "offset": 1898604032, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.15.attn_norm.weight", "offset": 1911506432, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1911526912, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.ffn_up.weight", "offset": 1950950912, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.15.ffn_norm.weight", "offset": 2011162112, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2011182592, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.15.attn_qkv.weight", "offset": 2022446592, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.16.attn_norm.weight", "offset": 2035348992, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2035369472, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.ffn_up.weight", "offset": 2074793472, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.16.ffn_norm.weight", "offset": 2135004672, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2135025152, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.16.attn_qkv.weight", "offset": 2146289152, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.17.attn_norm.weight", "offset": 2159191552, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2159212032, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.ffn_up.weight", "offset": 2198636032, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.17.ffn_norm.weight", "offset": 2258847232, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 2258867712, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.17.attn_qkv.weight", "offset": 2270131712, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.18.attn_norm.weight", "offset": 2283034112, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2283054592, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.ffn_up.weight", "offset": 2322478592, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.18.ffn_norm.weight", "offset": 2382689792, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 2382710272, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.18.attn_qkv.weight", "offset": 2393974272, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.19.attn_norm.weight", "offset": 2406876672, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2406897152, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.ffn_up.weight", "offset": 2446321152, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.19.ffn_norm.weight", "offset": 2506532352, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 2506552832, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.19.attn_qkv.weight", "offset": 2517816832, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.20.attn_norm.weight", "offset": 2530719232, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 2530739712, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.ffn_up.weight", "offset": 2570163712, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.20.ffn_norm.weight", "offset": 2630374912, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 2630395392, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.20.attn_qkv.weight", "offset": 2641659392, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.21.attn_output.weight", "offset": 2654561792, "shape": [ 5120, 5120 ], "size": 12902400, "type": 11 }, { "name": "blk.21.attn_qkv.weight", "offset": 2665825792, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.21.attn_norm.weight", "offset": 2678728192, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 2678748672, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.21.ffn_up.weight", "offset": 2718172672, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.21.ffn_norm.weight", "offset": 2778383872, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 2778404352, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 2778424832, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.ffn_up.weight", "offset": 2817848832, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.22.ffn_norm.weight", "offset": 2878060032, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 2878080512, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.22.attn_qkv.weight", "offset": 2889344512, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.23.attn_norm.weight", "offset": 2902246912, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 2902267392, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.ffn_up.weight", "offset": 2941691392, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.23.ffn_norm.weight", "offset": 3001902592, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 3001923072, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.23.attn_qkv.weight", "offset": 3013187072, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.24.attn_norm.weight", "offset": 3026089472, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 3026109952, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.ffn_up.weight", "offset": 3065533952, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.24.ffn_norm.weight", "offset": 3125745152, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 3125765632, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.24.attn_qkv.weight", "offset": 3137029632, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.25.attn_norm.weight", "offset": 3149932032, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 3149952512, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.ffn_up.weight", "offset": 3189376512, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.25.ffn_norm.weight", "offset": 3249587712, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 3249608192, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.25.attn_qkv.weight", "offset": 3260872192, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.26.attn_norm.weight", "offset": 3273774592, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 3273795072, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.ffn_up.weight", "offset": 3313219072, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.26.ffn_norm.weight", "offset": 3373430272, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 3373450752, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.26.attn_qkv.weight", "offset": 3384714752, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.27.attn_norm.weight", "offset": 3397617152, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 3397637632, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.ffn_up.weight", "offset": 3437061632, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.27.ffn_norm.weight", "offset": 3497272832, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 3497293312, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.27.attn_qkv.weight", "offset": 3508557312, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.28.attn_output.weight", "offset": 3521459712, "shape": [ 5120, 5120 ], "size": 12902400, "type": 11 }, { "name": "blk.28.attn_qkv.weight", "offset": 3532723712, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.28.attn_norm.weight", "offset": 3545626112, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 3545646592, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.28.ffn_up.weight", "offset": 3585070592, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.28.ffn_norm.weight", "offset": 3645281792, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 3645302272, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 3645322752, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.ffn_up.weight", "offset": 3684746752, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.29.ffn_norm.weight", "offset": 3744957952, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 3744978432, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.29.attn_qkv.weight", "offset": 3756242432, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.30.attn_norm.weight", "offset": 3769144832, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 3769165312, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.ffn_up.weight", "offset": 3808589312, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.30.ffn_norm.weight", "offset": 3868800512, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 3868820992, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.30.attn_qkv.weight", "offset": 3880084992, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.31.attn_norm.weight", "offset": 3892987392, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 3893007872, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.ffn_up.weight", "offset": 3932431872, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.31.ffn_norm.weight", "offset": 3992643072, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 3992663552, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.31.attn_qkv.weight", "offset": 4003927552, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.32.attn_norm.weight", "offset": 4016829952, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 4016850432, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.ffn_up.weight", "offset": 4056274432, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.32.ffn_norm.weight", "offset": 4116485632, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 4116506112, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.32.attn_qkv.weight", "offset": 4127770112, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.33.attn_norm.weight", "offset": 4140672512, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 4140692992, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.ffn_up.weight", "offset": 4180116992, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.33.ffn_norm.weight", "offset": 4240328192, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 4240348672, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.33.attn_qkv.weight", "offset": 4251612672, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.34.attn_norm.weight", "offset": 4264515072, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 4264535552, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.ffn_up.weight", "offset": 4303959552, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.34.ffn_norm.weight", "offset": 4364170752, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 4364191232, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.34.attn_qkv.weight", "offset": 4375455232, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.35.attn_output.weight", "offset": 4388357632, "shape": [ 5120, 5120 ], "size": 12902400, "type": 11 }, { "name": "blk.35.attn_qkv.weight", "offset": 4399621632, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "output.weight", "offset": 4412524032, "shape": [ 5120, 32064 ], "size": 12902400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 4547192832, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 4547213312, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.35.ffn_up.weight", "offset": 4586637312, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.35.ffn_norm.weight", "offset": 4646848512, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 4646868992, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 4646889472, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.ffn_up.weight", "offset": 4686313472, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.36.ffn_norm.weight", "offset": 4746524672, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 4746545152, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.36.attn_qkv.weight", "offset": 4757809152, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.37.attn_norm.weight", "offset": 4770711552, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 4770732032, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.ffn_up.weight", "offset": 4810156032, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.37.ffn_norm.weight", "offset": 4870367232, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 4870387712, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.37.attn_qkv.weight", "offset": 4881651712, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.38.attn_norm.weight", "offset": 4894554112, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 4894574592, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.ffn_up.weight", "offset": 4933998592, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.38.ffn_norm.weight", "offset": 4994209792, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 4994230272, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.38.attn_qkv.weight", "offset": 5005494272, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "blk.39.attn_norm.weight", "offset": 5018396672, "shape": [ 5120 ], "size": 12902400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 5018417152, "shape": [ 17920, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.ffn_up.weight", "offset": 5057841152, "shape": [ 5120, 35840 ], "size": 39424000, "type": 10 }, { "name": "blk.39.ffn_norm.weight", "offset": 5118052352, "shape": [ 5120 ], "size": 60211200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 5118072832, "shape": [ 5120, 5120 ], "size": 20480, "type": 11 }, { "name": "blk.39.attn_qkv.weight", "offset": 5129336832, "shape": [ 5120, 7680 ], "size": 11264000, "type": 10 }, { "name": "output_norm.weight", "offset": 5142239232, "shape": [ 5120 ], "size": 12902400, "type": 0 } ], "version": 3 }