phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 15, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 12 }, { "name": "blk.0.attn_norm.weight", "offset": 92344832, "shape": [ 5120 ], "size": 92344320, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 92365312, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 167629312, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 270848512, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 270868992, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.0.attn_qkv.weight", "offset": 285614592, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 312648192, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 312668672, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 387932672, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 491151872, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 491172352, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.1.attn_qkv.weight", "offset": 505917952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 532951552, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 532972032, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 608236032, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 711455232, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 711475712, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.attn_qkv.weight", "offset": 726221312, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 753254912, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 753275392, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 828539392, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 931758592, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 931779072, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.attn_qkv.weight", "offset": 946524672, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 973558272, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 973578752, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1048842752, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1152061952, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1152082432, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.attn_qkv.weight", "offset": 1166828032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1193861632, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1193882112, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1245491712, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1348710912, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1348731392, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.attn_qkv.weight", "offset": 1363476992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 1390510592, "shape": [ 5120, 35840 ], "size": 27033600, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 1493729792, "shape": [ 5120, 5120 ], "size": 103219200, "type": 12 }, { "name": "blk.6.attn_qkv.weight", "offset": 1508475392, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1535508992, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1535529472, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1587139072, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1690358272, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1690378752, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.attn_qkv.weight", "offset": 1705124352, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1732157952, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1732178432, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 1807442432, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1910661632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1910682112, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.attn_qkv.weight", "offset": 1925427712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1952461312, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1952481792, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 2004091392, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 2107310592, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2107331072, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.attn_qkv.weight", "offset": 2122076672, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 2149110272, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2149130752, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 2200740352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 2303959552, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2303980032, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.attn_qkv.weight", "offset": 2318725632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 2345759232, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2345779712, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 2421043712, "shape": [ 5120 ], "size": 75264000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2421064192, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2421084672, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2472694272, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2575913472, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2575933952, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.attn_qkv.weight", "offset": 2590679552, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 2617713152, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2617733632, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2669343232, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2772562432, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2772582912, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.attn_qkv.weight", "offset": 2787328512, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 2814362112, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2814382592, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 2889646592, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2992865792, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2992886272, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.attn_qkv.weight", "offset": 3007631872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 3034665472, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3034685952, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 3086295552, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 3189514752, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3189535232, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.attn_qkv.weight", "offset": 3204280832, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 3231314432, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3231334912, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 3282944512, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 3386163712, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3386184192, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.attn_qkv.weight", "offset": 3400929792, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 3427963392, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3427983872, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 3503247872, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 3606467072, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3606487552, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.attn_qkv.weight", "offset": 3621233152, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 3648266752, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3648287232, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 3699896832, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 3803116032, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3803136512, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.attn_qkv.weight", "offset": 3817882112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 3844915712, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3844936192, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3896545792, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 3999764992, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3999785472, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.attn_qkv.weight", "offset": 4014531072, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 4041564672, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4041585152, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 4116849152, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "\u0000\f\u0000\u0000\u0000\u0000r\ufffd\ufffd\u0000\u0000\u0000\u0000\u0016\u0000\u0000\u0000\u0000\u0000\u0000\u0000b", "offset": 4220068352, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4220088832, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.attn_qkv.weight", "offset": 4234834432, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 4261868032, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4261888512, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 4313498112, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 4416717312, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4416737792, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.attn_qkv.weight", "offset": 4431483392, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 4458516992, "shape": [ 5120, 5120 ], "size": 27033600, "type": 12 }, { "name": "blk.21.attn_qkv.weight", "offset": 4473262592, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 4500296192, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4500316672, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 4551926272, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 4655145472, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4655165952, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4655186432, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 4730450432, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 4833669632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4833690112, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.attn_qkv.weight", "offset": 4848435712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 4875469312, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4875489792, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 4927099392, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 5030318592, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5030339072, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.attn_qkv.weight", "offset": 5045084672, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 5072118272, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5072138752, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 5123748352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 5226967552, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5226988032, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.attn_qkv.weight", "offset": 5241733632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 5268767232, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5268787712, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 5344051712, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 5447270912, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5447291392, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.attn_qkv.weight", "offset": 5462036992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 5489070592, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5489091072, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 5540700672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 5643919872, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5643940352, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.attn_qkv.weight", "offset": 5658685952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 5685719552, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5685740032, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 5737349632, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 5840568832, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5840589312, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.attn_qkv.weight", "offset": 5855334912, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 5882368512, "shape": [ 5120, 5120 ], "size": 27033600, "type": 12 }, { "name": "blk.28.attn_qkv.weight", "offset": 5897114112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 5924147712, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5924168192, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 5999432192, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 6102651392, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 6102671872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6102692352, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 6154301952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 6257521152, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 6257541632, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.attn_qkv.weight", "offset": 6272287232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 6299320832, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6299341312, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 6350950912, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 6454170112, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 6454190592, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.attn_qkv.weight", "offset": 6468936192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 6495969792, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6495990272, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 6571254272, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 6674473472, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6674493952, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.attn_qkv.weight", "offset": 6689239552, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 6716273152, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6716293632, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 6767903232, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 6871122432, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6871142912, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.attn_qkv.weight", "offset": 6885888512, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 6912922112, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6912942592, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 6964552192, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 7067771392, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 7067791872, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.attn_qkv.weight", "offset": 7082537472, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 7109571072, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7109591552, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 7184855552, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 7288074752, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 7288095232, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.attn_qkv.weight", "offset": 7302840832, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 7329874432, "shape": [ 5120, 5120 ], "size": 27033600, "type": 12 }, { "name": "blk.35.attn_qkv.weight", "offset": 7344620032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "output.weight", "offset": 7371653632, "shape": [ 5120, 32064 ], "size": 27033600, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 7506322432, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7506342912, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 7581606912, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 7684826112, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7684846592, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7684867072, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 7760131072, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 7863350272, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 7863370752, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.attn_qkv.weight", "offset": 7878116352, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 7905149952, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7905170432, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 7980434432, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 8083653632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 8083674112, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.attn_qkv.weight", "offset": 8098419712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 8125453312, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8125473792, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 8200737792, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 8303956992, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 8303977472, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.attn_qkv.weight", "offset": 8318723072, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 8345756672, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8345777152, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 8421041152, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 8524260352, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 8524280832, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.attn_qkv.weight", "offset": 8539026432, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "output_norm.weight", "offset": 8566060032, "shape": [ 5120 ], "size": 27033600, "type": 0 } ], "version": 3 }