phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 18, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 14 }, { "name": "blk.0.attn_norm.weight", "offset": 134669312, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 134689792, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 209953792, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.0.ffn_norm.weight", "offset": 360481792, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 360502272, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.attn_qkv.weight", "offset": 382006272, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 414262272, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 414282752, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 489546752, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 640074752, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 640095232, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.attn_qkv.weight", "offset": 661599232, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 693855232, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 693875712, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 769139712, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 919667712, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 919688192, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.attn_qkv.weight", "offset": 941192192, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 973448192, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 973468672, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 1048732672, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 1199260672, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1199281152, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.attn_qkv.weight", "offset": 1220785152, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1253041152, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1253061632, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1328325632, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 1478853632, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1478874112, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.attn_qkv.weight", "offset": 1500378112, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1532634112, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1532654592, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 1607918592, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 1758446592, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1758467072, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.5.attn_qkv.weight", "offset": 1779971072, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 1812227072, "shape": [ 5120, 35840 ], "size": 32256000, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 1962755072, "shape": [ 5120, 5120 ], "size": 150528000, "type": 14 }, { "name": "blk.6.attn_qkv.weight", "offset": 1984259072, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 2016515072, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2016535552, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 2091799552, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 2242327552, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 2242348032, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.10.attn_qkv.weight", "offset": 2263852032, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 2296108032, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2296128512, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 2371392512, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 2521920512, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2521940992, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.attn_qkv.weight", "offset": 2543444992, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 2575700992, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2575721472, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 2650985472, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 2801513472, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2801533952, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.12.attn_qkv.weight", "offset": 2823037952, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 2855293952, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2855314432, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 2930578432, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 3081106432, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 3081126912, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.13.attn_qkv.weight", "offset": 3102630912, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 3134886912, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3134907392, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 3210171392, "shape": [ 5120 ], "size": 75264000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 3210191872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 3210212352, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 3285476352, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 3436004352, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 3436024832, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.7.attn_qkv.weight", "offset": 3457528832, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 3489784832, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3489805312, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 3565069312, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 3715597312, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3715617792, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.8.attn_qkv.weight", "offset": 3737121792, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 3769377792, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3769398272, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 3844662272, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.9.ffn_norm.weight", "offset": 3995190272, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3995210752, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.attn_qkv.weight", "offset": 4016714752, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 4048970752, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4048991232, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 4124255232, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 4274783232, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 4274803712, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.14.attn_qkv.weight", "offset": 4296307712, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 4328563712, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 4328584192, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 4403848192, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 4554376192, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 4554396672, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.15.attn_qkv.weight", "offset": 4575900672, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 4608156672, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4608177152, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 4683441152, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 4833969152, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4833989632, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.attn_qkv.weight", "offset": 4855493632, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 4887749632, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4887770112, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 4963034112, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 5113562112, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 5113582592, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.17.attn_qkv.weight", "offset": 5135086592, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 5167342592, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 5167363072, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 5242627072, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 5393155072, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 5393175552, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.18.attn_qkv.weight", "offset": 5414679552, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 5446935552, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 5446956032, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 5522220032, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 5672748032, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 5672768512, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.attn_qkv.weight", "offset": 5694272512, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 5726528512, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5726548992, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 5801812992, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 5952340992, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 5952361472, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.20.attn_qkv.weight", "offset": 5973865472, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 6006121472, "shape": [ 5120, 5120 ], "size": 32256000, "type": 14 }, { "name": "blk.21.attn_qkv.weight", "offset": 6027625472, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 6059881472, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 6059901952, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 6135165952, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 6285693952, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 6285714432, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 6285734912, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 6360998912, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 6511526912, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 6511547392, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.attn_qkv.weight", "offset": 6533051392, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 6565307392, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6565327872, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 6640591872, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 6791119872, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 6791140352, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.23.attn_qkv.weight", "offset": 6812644352, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 6844900352, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 6844920832, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 6920184832, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 7070712832, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 7070733312, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.24.attn_qkv.weight", "offset": 7092237312, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 7124493312, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 7124513792, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 7199777792, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 7350305792, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 7350326272, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.attn_qkv.weight", "offset": 7371830272, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 7404086272, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 7404106752, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 7479370752, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 7629898752, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 7629919232, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.26.attn_qkv.weight", "offset": 7651423232, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 7683679232, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 7683699712, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 7758963712, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.27.ffn_norm.weight", "offset": 7909491712, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 7909512192, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.27.attn_qkv.weight", "offset": 7931016192, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 7963272192, "shape": [ 5120, 5120 ], "size": 32256000, "type": 14 }, { "name": "blk.28.attn_qkv.weight", "offset": 7984776192, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 8017032192, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 8017052672, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 8092316672, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.28.ffn_norm.weight", "offset": 8242844672, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 8242865152, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 8242885632, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 8318149632, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.29.ffn_norm.weight", "offset": 8468677632, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 8468698112, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.29.attn_qkv.weight", "offset": 8490202112, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 8522458112, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 8522478592, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 8597742592, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.30.ffn_norm.weight", "offset": 8748270592, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 8748291072, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.30.attn_qkv.weight", "offset": 8769795072, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 8802051072, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 8802071552, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 8877335552, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.31.ffn_norm.weight", "offset": 9027863552, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 9027884032, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.31.attn_qkv.weight", "offset": 9049388032, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 9081644032, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 9081664512, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.32.ffn_up.weight", "offset": 9156928512, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.32.ffn_norm.weight", "offset": 9307456512, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 9307476992, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.32.attn_qkv.weight", "offset": 9328980992, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 9361236992, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 9361257472, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.33.ffn_up.weight", "offset": 9436521472, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.33.ffn_norm.weight", "offset": 9587049472, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 9587069952, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.33.attn_qkv.weight", "offset": 9608573952, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 9640829952, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 9640850432, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 9716114432, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.34.ffn_norm.weight", "offset": 9866642432, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 9866662912, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.attn_qkv.weight", "offset": 9888166912, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 9920422912, "shape": [ 5120, 5120 ], "size": 32256000, "type": 14 }, { "name": "blk.35.attn_qkv.weight", "offset": 9941926912, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "output.weight", "offset": 9974182912, "shape": [ 5120, 32064 ], "size": 32256000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 10108851712, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 10108872192, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 10184136192, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.35.ffn_norm.weight", "offset": 10334664192, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 10334684672, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 10334705152, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 10409969152, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.36.ffn_norm.weight", "offset": 10560497152, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 10560517632, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.attn_qkv.weight", "offset": 10582021632, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 10614277632, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 10614298112, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 10689562112, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.37.ffn_norm.weight", "offset": 10840090112, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 10840110592, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.attn_qkv.weight", "offset": 10861614592, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 10893870592, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 10893891072, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 10969155072, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.38.ffn_norm.weight", "offset": 11119683072, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 11119703552, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.attn_qkv.weight", "offset": 11141207552, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 11173463552, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 11173484032, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 11248748032, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.39.ffn_norm.weight", "offset": 11399276032, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 11399296512, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.attn_qkv.weight", "offset": 11420800512, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "output_norm.weight", "offset": 11453056512, "shape": [ 5120 ], "size": 32256000, "type": 0 } ], "version": 3 }