phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 2, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 2 }, { "name": "blk.0.attn_norm.weight", "offset": 92344832, "shape": [ 5120 ], "size": 92344320, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 92365312, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 143974912, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 247194112, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 247214592, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.0.attn_qkv.weight", "offset": 261960192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 284078592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 284099072, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 335708672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 438927872, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 438948352, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.1.attn_qkv.weight", "offset": 453693952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 475812352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 475832832, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 527442432, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 630661632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 630682112, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.2.attn_qkv.weight", "offset": 645427712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 667546112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 667566592, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 719176192, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 822395392, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 822415872, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.3.attn_qkv.weight", "offset": 837161472, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 859279872, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 859300352, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 910909952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 1014129152, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1014149632, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.4.attn_qkv.weight", "offset": 1028895232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 1051013632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1051034112, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 1102643712, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1205862912, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1205883392, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.5.attn_qkv.weight", "offset": 1220628992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 1242747392, "shape": [ 5120, 35840 ], "size": 22118400, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1345966592, "shape": [ 5120, 5120 ], "size": 103219200, "type": 2 }, { "name": "blk.6.attn_qkv.weight", "offset": 1360712192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1382830592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1382851072, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 1434460672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1537679872, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1537700352, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.10.attn_qkv.weight", "offset": 1552445952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 1574564352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1574584832, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1626194432, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 1729413632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1729434112, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.11.attn_qkv.weight", "offset": 1744179712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1766298112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1766318592, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 1817928192, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 1921147392, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1921167872, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.12.attn_qkv.weight", "offset": 1935913472, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 1958031872, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1958052352, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 2009661952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 2112881152, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2112901632, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.13.attn_qkv.weight", "offset": 2127647232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 2149765632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2149786112, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 2201395712, "shape": [ 5120 ], "size": 51609600, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2201416192, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2201436672, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 2253046272, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 2356265472, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2356285952, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.7.attn_qkv.weight", "offset": 2371031552, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 2393149952, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2393170432, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 2444780032, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 2547999232, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2548019712, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.8.attn_qkv.weight", "offset": 2562765312, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 2584883712, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2584904192, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 2636513792, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 2739732992, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2739753472, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.9.attn_qkv.weight", "offset": 2754499072, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 2776617472, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2776637952, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 2828247552, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 2931466752, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2931487232, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.14.attn_qkv.weight", "offset": 2946232832, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 2968351232, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2968371712, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 3019981312, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 3123200512, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3123220992, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.15.attn_qkv.weight", "offset": 3137966592, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 3160084992, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3160105472, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 3211715072, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 3314934272, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3314954752, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.16.attn_qkv.weight", "offset": 3329700352, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 3351818752, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3351839232, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 3403448832, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 3506668032, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3506688512, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.17.attn_qkv.weight", "offset": 3521434112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 3543552512, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3543572992, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 3595182592, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 3698401792, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3698422272, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.18.attn_qkv.weight", "offset": 3713167872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 3735286272, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3735306752, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 3786916352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "output.weight\u0002\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000", "offset": 3890135552, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3890156032, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.19.attn_qkv.weight", "offset": 3904901632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 3927020032, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3927040512, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 3978650112, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 4081869312, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4081889792, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.20.attn_qkv.weight", "offset": 4096635392, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 4118753792, "shape": [ 5120, 5120 ], "size": 22118400, "type": 2 }, { "name": "blk.21.attn_qkv.weight", "offset": 4133499392, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 4155617792, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4155638272, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 4207247872, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 4310467072, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4310487552, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4310508032, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 4362117632, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 4465336832, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4465357312, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.22.attn_qkv.weight", "offset": 4480102912, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 4502221312, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4502241792, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 4553851392, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 4657070592, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4657091072, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.23.attn_qkv.weight", "offset": 4671836672, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 4693955072, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4693975552, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 4745585152, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 4848804352, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4848824832, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.24.attn_qkv.weight", "offset": 4863570432, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 4885688832, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4885709312, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 4937318912, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 5040538112, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5040558592, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.25.attn_qkv.weight", "offset": 5055304192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 5077422592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5077443072, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 5129052672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 5232271872, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5232292352, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.26.attn_qkv.weight", "offset": 5247037952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 5269156352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5269176832, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 5320786432, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 5424005632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5424026112, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.27.attn_qkv.weight", "offset": 5438771712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 5460890112, "shape": [ 5120, 5120 ], "size": 22118400, "type": 2 }, { "name": "blk.28.attn_qkv.weight", "offset": 5475635712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 5497754112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5497774592, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 5549384192, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 5652603392, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 5652623872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5652644352, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 5704253952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 5807473152, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5807493632, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.29.attn_qkv.weight", "offset": 5822239232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 5844357632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5844378112, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 5895987712, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 5999206912, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 5999227392, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.30.attn_qkv.weight", "offset": 6013972992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 6036091392, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6036111872, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 6087721472, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 6190940672, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6190961152, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.31.attn_qkv.weight", "offset": 6205706752, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 6227825152, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6227845632, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.32.ffn_up.weight", "offset": 6279455232, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.32.ffn_norm.weight", "offset": 6382674432, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6382694912, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.32.attn_qkv.weight", "offset": 6397440512, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.33.attn_norm.weight", "offset": 6419558912, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6419579392, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.33.ffn_up.weight", "offset": 6471188992, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.33.ffn_norm.weight", "offset": 6574408192, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 6574428672, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.33.attn_qkv.weight", "offset": 6589174272, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 6611292672, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6611313152, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.34.ffn_up.weight", "offset": 6662922752, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.34.ffn_norm.weight", "offset": 6766141952, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 6766162432, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.34.attn_qkv.weight", "offset": 6780908032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.35.attn_output.weight", "offset": 6803026432, "shape": [ 5120, 5120 ], "size": 22118400, "type": 2 }, { "name": "blk.35.attn_qkv.weight", "offset": 6817772032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "output.weight", "offset": 6839890432, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 6974559232, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6974579712, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.35.ffn_up.weight", "offset": 7026189312, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.35.ffn_norm.weight", "offset": 7129408512, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7129428992, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7129449472, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.36.ffn_up.weight", "offset": 7181059072, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.36.ffn_norm.weight", "offset": 7284278272, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 7284298752, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.36.attn_qkv.weight", "offset": 7299044352, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.37.attn_norm.weight", "offset": 7321162752, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7321183232, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.37.ffn_up.weight", "offset": 7372792832, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.37.ffn_norm.weight", "offset": 7476012032, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 7476032512, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.37.attn_qkv.weight", "offset": 7490778112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.38.attn_norm.weight", "offset": 7512896512, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7512916992, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.38.ffn_up.weight", "offset": 7564526592, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.38.ffn_norm.weight", "offset": 7667745792, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 7667766272, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.38.attn_qkv.weight", "offset": 7682511872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 7704630272, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7704650752, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.39.ffn_up.weight", "offset": 7756260352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.39.ffn_norm.weight", "offset": 7859479552, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 7859500032, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.39.attn_qkv.weight", "offset": 7874245632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "output_norm.weight", "offset": 7896364032, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }