phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 17, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 13 }, { "name": "blk.0.attn_norm.weight", "offset": 112865792, "shape": [ 5120 ], "size": 112865280, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 112886272, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 188150272, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 314307072, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 314327552, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.attn_qkv.weight", "offset": 332349952, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 364605952, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 364626432, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 439890432, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 566047232, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 566067712, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.attn_qkv.weight", "offset": 584090112, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 616346112, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 616366592, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 691630592, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 817787392, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 817807872, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.attn_qkv.weight", "offset": 835830272, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 868086272, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 868106752, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 943370752, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 1069527552, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1069548032, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.attn_qkv.weight", "offset": 1087570432, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1119826432, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1119846912, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1195110912, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 1321267712, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1321288192, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.attn_qkv.weight", "offset": 1339310592, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1371566592, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1371587072, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1434665472, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 1560822272, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1560842752, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.attn_qkv.weight", "offset": 1578865152, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 1611121152, "shape": [ 5120, 35840 ], "size": 32256000, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 1737277952, "shape": [ 5120, 5120 ], "size": 126156800, "type": 13 }, { "name": "blk.6.attn_qkv.weight", "offset": 1755300352, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 1787556352, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1787576832, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 1850655232, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 1976812032, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1976832512, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.attn_qkv.weight", "offset": 1994854912, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 2027110912, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2027131392, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "\ufffd\u0001\u0000\u0000\u0000\u0016\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.30.", "offset": 2102395392, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 2228552192, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2228572672, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.11.attn_qkv.weight", "offset": 2246595072, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 2278851072, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2278871552, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 2341949952, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 2468106752, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2468127232, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.attn_qkv.weight", "offset": 2486149632, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 2518405632, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2518426112, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 2581504512, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 2707661312, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2707681792, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.attn_qkv.weight", "offset": 2725704192, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 2757960192, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2757980672, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 2833244672, "shape": [ 5120 ], "size": 75264000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2833265152, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2833285632, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 2896364032, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 3022520832, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 3022541312, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.attn_qkv.weight", "offset": 3040563712, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 3072819712, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3072840192, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 3135918592, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 3262075392, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3262095872, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.attn_qkv.weight", "offset": 3280118272, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 3312374272, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3312394752, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 3387658752, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 3513815552, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3513836032, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.9.attn_qkv.weight", "offset": 3531858432, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 3564114432, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3564134912, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 3627213312, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 3753370112, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3753390592, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.attn_qkv.weight", "offset": 3771412992, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 3803668992, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3803689472, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 3866767872, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 3992924672, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3992945152, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.attn_qkv.weight", "offset": 4010967552, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 4043223552, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4043244032, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 4118508032, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 4244664832, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4244685312, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.16.attn_qkv.weight", "offset": 4262707712, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 4294963712, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4294984192, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 4358062592, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 4484219392, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 4484239872, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.attn_qkv.weight", "offset": 4502262272, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 4534518272, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4534538752, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 4597617152, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 4723773952, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4723794432, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.attn_qkv.weight", "offset": 4741816832, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 4774072832, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4774093312, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 4849357312, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 4975514112, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4975534592, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.19.attn_qkv.weight", "offset": 4993556992, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 5025812992, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5025833472, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 5088911872, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 5215068672, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 5215089152, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.attn_qkv.weight", "offset": 5233111552, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 5265367552, "shape": [ 5120, 5120 ], "size": 32256000, "type": 13 }, { "name": "blk.21.attn_qkv.weight", "offset": 5283389952, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 5315645952, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5315666432, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 5378744832, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 5504901632, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 5504922112, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5504942592, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 5580206592, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 5706363392, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5706383872, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.22.attn_qkv.weight", "offset": 5724406272, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 5756662272, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5756682752, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 5819761152, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 5945917952, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5945938432, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.attn_qkv.weight", "offset": 5963960832, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 5996216832, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5996237312, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 6059315712, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 6185472512, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 6185492992, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.attn_qkv.weight", "offset": 6203515392, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 6235771392, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6235791872, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 6311055872, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 6437212672, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6437233152, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.25.attn_qkv.weight", "offset": 6455255552, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 6487511552, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6487532032, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 6550610432, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 6676767232, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6676787712, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.attn_qkv.weight", "offset": 6694810112, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 6727066112, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6727086592, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 6790164992, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 6916321792, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6916342272, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.attn_qkv.weight", "offset": 6934364672, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 6966620672, "shape": [ 5120, 5120 ], "size": 32256000, "type": 13 }, { "name": "blk.28.attn_qkv.weight", "offset": 6984643072, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 7016899072, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 7016919552, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 7092183552, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 7218340352, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 7218360832, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 7218381312, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 7281459712, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 7407616512, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7407636992, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.attn_qkv.weight", "offset": 7425659392, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 7457915392, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7457935872, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 7521014272, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 7647171072, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7647191552, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.attn_qkv.weight", "offset": 7665213952, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 7697469952, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7697490432, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 7772754432, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 7898911232, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7898931712, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.31.attn_qkv.weight", "offset": 7916954112, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 7949210112, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7949230592, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 8012308992, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 8138465792, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 8138486272, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.attn_qkv.weight", "offset": 8156508672, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 8188764672, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 8188785152, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 8251863552, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 8378020352, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 8378040832, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.attn_qkv.weight", "offset": 8396063232, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 8428319232, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8428339712, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 8503603712, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 8629760512, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 8629780992, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.34.attn_qkv.weight", "offset": 8647803392, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 8680059392, "shape": [ 5120, 5120 ], "size": 32256000, "type": 13 }, { "name": "blk.35.attn_qkv.weight", "offset": 8698081792, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "output.weight", "offset": 8730337792, "shape": [ 5120, 32064 ], "size": 32256000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 8865006592, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 8865027072, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 8940291072, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 9066447872, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 9066468352, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 9066488832, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 9141752832, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 9267909632, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 9267930112, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.36.attn_qkv.weight", "offset": 9285952512, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 9318208512, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 9318228992, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 9393492992, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 9519649792, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 9519670272, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.37.attn_qkv.weight", "offset": 9537692672, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 9569948672, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9569969152, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 9645233152, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 9771389952, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 9771410432, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.38.attn_qkv.weight", "offset": 9789432832, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 9821688832, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 9821709312, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 9896973312, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 10023130112, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 10023150592, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.39.attn_qkv.weight", "offset": 10041172992, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "output_norm.weight", "offset": 10073428992, "shape": [ 5120 ], "size": 32256000, "type": 0 } ], "version": 3 }