phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 8, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 6 }, { "name": "blk.0.attn_norm.weight", "offset": 112865792, "shape": [ 5120 ], "size": 112865280, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 112886272, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 175964672, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 302121472, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 302141952, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.0.attn_qkv.weight", "offset": 320164352, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 347197952, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 347218432, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 410296832, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 536453632, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 536474112, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.1.attn_qkv.weight", "offset": 554496512, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 581530112, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 581550592, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 644628992, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 770785792, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 770806272, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.2.attn_qkv.weight", "offset": 788828672, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 815862272, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 815882752, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 878961152, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 1005117952, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1005138432, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.3.attn_qkv.weight", "offset": 1023160832, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 1050194432, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1050214912, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 1113293312, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 1239450112, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1239470592, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.4.attn_qkv.weight", "offset": 1257492992, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 1284526592, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1284547072, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 1347625472, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 1473782272, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1473802752, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.5.attn_qkv.weight", "offset": 1491825152, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 1518858752, "shape": [ 5120, 35840 ], "size": 27033600, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 1645015552, "shape": [ 5120, 5120 ], "size": 126156800, "type": 6 }, { "name": "blk.6.attn_qkv.weight", "offset": 1663037952, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 1690071552, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1690092032, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 1753170432, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 1879327232, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1879347712, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.10.attn_qkv.weight", "offset": 1897370112, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 1924403712, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1924424192, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 1987502592, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 2113659392, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2113679872, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.11.attn_qkv.weight", "offset": 2131702272, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 2158735872, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2158756352, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 2221834752, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 2347991552, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2348012032, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.12.attn_qkv.weight", "offset": 2366034432, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 2393068032, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2393088512, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 2456166912, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 2582323712, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2582344192, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.13.attn_qkv.weight", "offset": 2600366592, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 2627400192, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2627420672, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 2690499072, "shape": [ 5120 ], "size": 63078400, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2690519552, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2690540032, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 2753618432, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 2879775232, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2879795712, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.7.attn_qkv.weight", "offset": 2897818112, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 2924851712, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2924872192, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 2987950592, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 3114107392, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3114127872, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.8.attn_qkv.weight", "offset": 3132150272, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 3159183872, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3159204352, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 3222282752, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 3348439552, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3348460032, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.9.attn_qkv.weight", "offset": 3366482432, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 3393516032, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3393536512, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 3456614912, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 3582771712, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3582792192, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.14.attn_qkv.weight", "offset": 3600814592, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 3627848192, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3627868672, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 3690947072, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 3817103872, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3817124352, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.15.attn_qkv.weight", "offset": 3835146752, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 3862180352, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3862200832, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 3925279232, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 4051436032, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4051456512, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.16.attn_qkv.weight", "offset": 4069478912, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 4096512512, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4096532992, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 4159611392, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 4285768192, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 4285788672, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.17.attn_qkv.weight", "offset": 4303811072, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 4330844672, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4330865152, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 4393943552, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 4520100352, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4520120832, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.18.attn_qkv.weight", "offset": 4538143232, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 4565176832, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4565197312, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 4628275712, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "output.weight\u0002\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000", "offset": 4754432512, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4754452992, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.19.attn_qkv.weight", "offset": 4772475392, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 4799508992, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4799529472, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 4862607872, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 4988764672, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4988785152, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.20.attn_qkv.weight", "offset": 5006807552, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 5033841152, "shape": [ 5120, 5120 ], "size": 27033600, "type": 6 }, { "name": "blk.21.attn_qkv.weight", "offset": 5051863552, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 5078897152, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5078917632, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 5141996032, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 5268152832, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 5268173312, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5268193792, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 5331272192, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 5457428992, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5457449472, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.22.attn_qkv.weight", "offset": 5475471872, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 5502505472, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5502525952, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 5565604352, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 5691761152, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5691781632, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.23.attn_qkv.weight", "offset": 5709804032, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.24.attn_norm.weight", "offset": 5736837632, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5736858112, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.24.ffn_up.weight", "offset": 5799936512, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.24.ffn_norm.weight", "offset": 5926093312, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5926113792, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.24.attn_qkv.weight", "offset": 5944136192, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.25.attn_norm.weight", "offset": 5971169792, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5971190272, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 6034268672, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.25.ffn_norm.weight", "offset": 6160425472, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6160445952, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.25.attn_qkv.weight", "offset": 6178468352, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.26.attn_norm.weight", "offset": 6205501952, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6205522432, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.26.ffn_up.weight", "offset": 6268600832, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.26.ffn_norm.weight", "offset": 6394757632, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6394778112, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.26.attn_qkv.weight", "offset": 6412800512, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.27.attn_norm.weight", "offset": 6439834112, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6439854592, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.27.ffn_up.weight", "offset": 6502932992, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.27.ffn_norm.weight", "offset": 6629089792, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6629110272, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.27.attn_qkv.weight", "offset": 6647132672, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.28.attn_output.weight", "offset": 6674166272, "shape": [ 5120, 5120 ], "size": 27033600, "type": 6 }, { "name": "blk.28.attn_qkv.weight", "offset": 6692188672, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.28.attn_norm.weight", "offset": 6719222272, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6719242752, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.28.ffn_up.weight", "offset": 6782321152, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.28.ffn_norm.weight", "offset": 6908477952, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 6908498432, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6908518912, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.29.ffn_up.weight", "offset": 6971597312, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.29.ffn_norm.weight", "offset": 7097754112, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7097774592, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.29.attn_qkv.weight", "offset": 7115796992, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.30.attn_norm.weight", "offset": 7142830592, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7142851072, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.30.ffn_up.weight", "offset": 7205929472, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.30.ffn_norm.weight", "offset": 7332086272, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7332106752, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.30.attn_qkv.weight", "offset": 7350129152, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.31.attn_norm.weight", "offset": 7377162752, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7377183232, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.31.ffn_up.weight", "offset": 7440261632, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.31.ffn_norm.weight", "offset": 7566418432, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7566438912, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.31.attn_qkv.weight", "offset": 7584461312, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.32.attn_norm.weight", "offset": 7611494912, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7611515392, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.32.ffn_up.weight", "offset": 7674593792, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.32.ffn_norm.weight", "offset": 7800750592, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 7800771072, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.32.attn_qkv.weight", "offset": 7818793472, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.33.attn_norm.weight", "offset": 7845827072, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 7845847552, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.33.ffn_up.weight", "offset": 7908925952, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.33.ffn_norm.weight", "offset": 8035082752, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 8035103232, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.33.attn_qkv.weight", "offset": 8053125632, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.34.attn_norm.weight", "offset": 8080159232, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8080179712, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.34.ffn_up.weight", "offset": 8143258112, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.34.ffn_norm.weight", "offset": 8269414912, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 8269435392, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.34.attn_qkv.weight", "offset": 8287457792, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.35.attn_output.weight", "offset": 8314491392, "shape": [ 5120, 5120 ], "size": 27033600, "type": 6 }, { "name": "blk.35.attn_qkv.weight", "offset": 8332513792, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "output.weight", "offset": 8359547392, "shape": [ 5120, 32064 ], "size": 27033600, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 8494216192, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 8494236672, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.35.ffn_up.weight", "offset": 8557315072, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.35.ffn_norm.weight", "offset": 8683471872, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 8683492352, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 8683512832, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.36.ffn_up.weight", "offset": 8746591232, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.36.ffn_norm.weight", "offset": 8872748032, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 8872768512, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.36.attn_qkv.weight", "offset": 8890790912, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.37.attn_norm.weight", "offset": 8917824512, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 8917844992, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.37.ffn_up.weight", "offset": 8980923392, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.37.ffn_norm.weight", "offset": 9107080192, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 9107100672, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.37.attn_qkv.weight", "offset": 9125123072, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.38.attn_norm.weight", "offset": 9152156672, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9152177152, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.38.ffn_up.weight", "offset": 9215255552, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.38.ffn_norm.weight", "offset": 9341412352, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 9341432832, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.38.attn_qkv.weight", "offset": 9359455232, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.39.attn_norm.weight", "offset": 9386488832, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 9386509312, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.39.ffn_up.weight", "offset": 9449587712, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.39.ffn_norm.weight", "offset": 9575744512, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 9575764992, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.39.attn_qkv.weight", "offset": 9593787392, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "output_norm.weight", "offset": 9620820992, "shape": [ 5120 ], "size": 27033600, "type": 0 } ], "version": 3 }