phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 15, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 92344320, "shape": [ 5120 ], "size": 92344320, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 92364800, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 167628800, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 270848000, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 270868480, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.0.attn_qkv.weight", "offset": 285614080, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.1.attn_norm.weight", "offset": 312647680, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 312668160, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 387932160, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 491151360, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 491171840, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.1.attn_qkv.weight", "offset": 505917440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.2.attn_norm.weight", "offset": 532951040, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 532971520, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 608235520, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 711454720, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 711475200, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.attn_qkv.weight", "offset": 726220800, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.3.attn_norm.weight", "offset": 753254400, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 753274880, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 828538880, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 931758080, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 931778560, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.attn_qkv.weight", "offset": 946524160, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.4.attn_norm.weight", "offset": 973557760, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 973578240, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1048842240, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1152061440, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1152081920, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.attn_qkv.weight", "offset": 1166827520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.5.attn_norm.weight", "offset": 1193861120, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1193881600, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1245491200, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1348710400, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1348730880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.attn_qkv.weight", "offset": 1363476480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.6.ffn_up.weight", "offset": 1390510080, "shape": [ 5120, 35840 ], "size": 27033600, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 1493729280, "shape": [ 5120, 5120 ], "size": 103219200, "type": 12 }, { "name": "blk.6.attn_qkv.weight", "offset": 1508474880, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.10.attn_norm.weight", "offset": 1535508480, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1535528960, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1587138560, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1690357760, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1690378240, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.attn_qkv.weight", "offset": 1705123840, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.11.attn_norm.weight", "offset": 1732157440, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1732177920, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 1807441920, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1910661120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1910681600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.attn_qkv.weight", "offset": 1925427200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.12.attn_norm.weight", "offset": 1952460800, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1952481280, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 2004090880, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 2107310080, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2107330560, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.attn_qkv.weight", "offset": 2122076160, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.13.attn_norm.weight", "offset": 2149109760, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2149130240, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 2200739840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 2303959040, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2303979520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.attn_qkv.weight", "offset": 2318725120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.6.attn_norm.weight", "offset": 2345758720, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2345779200, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 2421043200, "shape": [ 5120 ], "size": 75264000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2421063680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2421084160, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2472693760, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2575912960, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2575933440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.attn_qkv.weight", "offset": 2590679040, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.8.attn_norm.weight", "offset": 2617712640, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2617733120, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2669342720, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2772561920, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2772582400, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.attn_qkv.weight", "offset": 2787328000, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.9.attn_norm.weight", "offset": 2814361600, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2814382080, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 2889646080, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2992865280, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2992885760, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.attn_qkv.weight", "offset": 3007631360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.14.attn_norm.weight", "offset": 3034664960, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3034685440, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 3086295040, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 3189514240, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3189534720, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.attn_qkv.weight", "offset": 3204280320, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.15.attn_norm.weight", "offset": 3231313920, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3231334400, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 3282944000, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 3386163200, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3386183680, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.attn_qkv.weight", "offset": 3400929280, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.16.attn_norm.weight", "offset": 3427962880, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3427983360, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 3503247360, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 3606466560, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3606487040, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.attn_qkv.weight", "offset": 3621232640, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.17.attn_norm.weight", "offset": 3648266240, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3648286720, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 3699896320, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 3803115520, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3803136000, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.attn_qkv.weight", "offset": 3817881600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.18.attn_norm.weight", "offset": 3844915200, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3844935680, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3896545280, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 3999764480, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3999784960, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.attn_qkv.weight", "offset": 4014530560, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.19.attn_norm.weight", "offset": 4041564160, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4041584640, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 4116848640, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 4220067840, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4220088320, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.attn_qkv.weight", "offset": 4234833920, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.20.attn_norm.weight", "offset": 4261867520, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4261888000, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 4313497600, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 4416716800, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4416737280, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.attn_qkv.weight", "offset": 4431482880, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.21.attn_output.weight", "offset": 4458516480, "shape": [ 5120, 5120 ], "size": 27033600, "type": 12 }, { "name": "blk.21.attn_qkv.weight", "offset": 4473262080, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.21.attn_norm.weight", "offset": 4500295680, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4500316160, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 4551925760, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 4655144960, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4655165440, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4655185920, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 4730449920, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 4833669120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4833689600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.attn_qkv.weight", "offset": 4848435200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.23.attn_norm.weight", "offset": 4875468800, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4875489280, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 4927098880, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 5030318080, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5030338560, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.attn_qkv.weight", "offset": 5045084160, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.24.attn_norm.weight", "offset": 5072117760, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5072138240, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 5123747840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 5226967040, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5226987520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.attn_qkv.weight", "offset": 5241733120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.25.attn_norm.weight", "offset": 5268766720, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5268787200, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 5344051200, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 5447270400, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5447290880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.attn_qkv.weight", "offset": 5462036480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.26.attn_norm.weight", "offset": 5489070080, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5489090560, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 5540700160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 5643919360, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5643939840, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.attn_qkv.weight", "offset": 5658685440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.27.attn_norm.weight", "offset": 5685719040, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5685739520, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 5737349120, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 5840568320, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5840588800, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.attn_qkv.weight", "offset": 5855334400, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.28.attn_output.weight", "offset": 5882368000, "shape": [ 5120, 5120 ], "size": 27033600, "type": 12 }, { "name": "blk.28.attn_qkv.weight", "offset": 5897113600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.28.attn_norm.weight", "offset": 5924147200, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5924167680, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 5999431680, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 6102650880, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 6102671360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6102691840, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 6154301440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 6257520640, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 6257541120, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.attn_qkv.weight", "offset": 6272286720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.30.attn_norm.weight", "offset": 6299320320, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6299340800, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 6350950400, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 6454169600, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 6454190080, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.attn_qkv.weight", "offset": 6468935680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.31.attn_norm.weight", "offset": 6495969280, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6495989760, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "\ufffd.wKi\ufffd\ufffd\ufffd\ufffd\ufffd\u0005\ufffdgF\ufffd'p\ufffd[\ufffd", "offset": 6571253760, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 6674472960, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6674493440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.attn_qkv.weight", "offset": 6689239040, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.32.attn_norm.weight", "offset": 6716272640, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6716293120, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 6767902720, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 6871121920, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6871142400, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.attn_qkv.weight", "offset": 6885888000, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.33.attn_norm.weight", "offset": 6912921600, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6912942080, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 6964551680, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 7067770880, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 7067791360, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.attn_qkv.weight", "offset": 7082536960, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.34.attn_norm.weight", "offset": 7109570560, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7109591040, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 7184855040, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 7288074240, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 7288094720, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.attn_qkv.weight", "offset": 7302840320, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.35.attn_output.weight", "offset": 7329873920, "shape": [ 5120, 5120 ], "size": 27033600, "type": 12 }, { "name": "blk.35.attn_qkv.weight", "offset": 7344619520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "output.weight", "offset": 7371653120, "shape": [ 5120, 32064 ], "size": 27033600, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 7506321920, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7506342400, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 7581606400, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 7684825600, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7684846080, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7684866560, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 7760130560, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 7863349760, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 7863370240, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.attn_qkv.weight", "offset": 7878115840, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.37.attn_norm.weight", "offset": 7905149440, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7905169920, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 7980433920, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 8083653120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 8083673600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.attn_qkv.weight", "offset": 8098419200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.38.attn_norm.weight", "offset": 8125452800, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8125473280, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 8200737280, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 8303956480, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 8303976960, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.attn_qkv.weight", "offset": 8318722560, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "blk.39.attn_norm.weight", "offset": 8345756160, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8345776640, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 8421040640, "shape": [ 5120, 35840 ], "size": 75264000, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 8524259840, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 8524280320, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.attn_qkv.weight", "offset": 8539025920, "shape": [ 5120, 7680 ], "size": 14745600, "type": 13 }, { "name": "output_norm.weight", "offset": 8566059520, "shape": [ 5120 ], "size": 27033600, "type": 0 } ], "version": 3 }