phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 8, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 112865280, "shape": [ 5120 ], "size": 112865280, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 112885760, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.0.ffn_up.weight", "offset": 175964160, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.0.ffn_norm.weight", "offset": 302120960, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 302141440, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.0.attn_qkv.weight", "offset": 320163840, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.1.attn_norm.weight", "offset": 347197440, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 347217920, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.1.ffn_up.weight", "offset": 410296320, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.1.ffn_norm.weight", "offset": 536453120, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 536473600, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.1.attn_qkv.weight", "offset": 554496000, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.2.attn_norm.weight", "offset": 581529600, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 581550080, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.2.ffn_up.weight", "offset": 644628480, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.2.ffn_norm.weight", "offset": 770785280, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 770805760, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.2.attn_qkv.weight", "offset": 788828160, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.3.attn_norm.weight", "offset": 815861760, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 815882240, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.3.ffn_up.weight", "offset": 878960640, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.3.ffn_norm.weight", "offset": 1005117440, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1005137920, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.3.attn_qkv.weight", "offset": 1023160320, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.4.attn_norm.weight", "offset": 1050193920, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1050214400, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.4.ffn_up.weight", "offset": 1113292800, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.4.ffn_norm.weight", "offset": 1239449600, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1239470080, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.4.attn_qkv.weight", "offset": 1257492480, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.5.attn_norm.weight", "offset": 1284526080, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1284546560, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.5.ffn_up.weight", "offset": 1347624960, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.5.ffn_norm.weight", "offset": 1473781760, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1473802240, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.5.attn_qkv.weight", "offset": 1491824640, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.6.ffn_up.weight", "offset": 1518858240, "shape": [ 5120, 35840 ], "size": 27033600, "type": 6 }, { "name": "blk.6.attn_output.weight", "offset": 1645015040, "shape": [ 5120, 5120 ], "size": 126156800, "type": 6 }, { "name": "blk.6.attn_qkv.weight", "offset": 1663037440, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.10.attn_norm.weight", "offset": 1690071040, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1690091520, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.10.ffn_up.weight", "offset": 1753169920, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.10.ffn_norm.weight", "offset": 1879326720, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1879347200, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.10.attn_qkv.weight", "offset": 1897369600, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.11.attn_norm.weight", "offset": 1924403200, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1924423680, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.11.ffn_up.weight", "offset": 1987502080, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.11.ffn_norm.weight", "offset": 2113658880, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2113679360, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.11.attn_qkv.weight", "offset": 2131701760, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.12.attn_norm.weight", "offset": 2158735360, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2158755840, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.12.ffn_up.weight", "offset": 2221834240, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.12.ffn_norm.weight", "offset": 2347991040, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2348011520, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.12.attn_qkv.weight", "offset": 2366033920, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.13.attn_norm.weight", "offset": 2393067520, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2393088000, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.13.ffn_up.weight", "offset": 2456166400, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.13.ffn_norm.weight", "offset": 2582323200, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2582343680, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.13.attn_qkv.weight", "offset": 2600366080, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.6.attn_norm.weight", "offset": 2627399680, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2627420160, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.6.ffn_norm.weight", "offset": 2690498560, "shape": [ 5120 ], "size": 63078400, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2690519040, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2690539520, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.7.ffn_up.weight", "offset": 2753617920, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.7.ffn_norm.weight", "offset": 2879774720, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2879795200, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.7.attn_qkv.weight", "offset": 2897817600, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.8.attn_norm.weight", "offset": 2924851200, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2924871680, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.8.ffn_up.weight", "offset": 2987950080, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.8.ffn_norm.weight", "offset": 3114106880, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3114127360, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.8.attn_qkv.weight", "offset": 3132149760, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.9.attn_norm.weight", "offset": 3159183360, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3159203840, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.9.ffn_up.weight", "offset": 3222282240, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.9.ffn_norm.weight", "offset": 3348439040, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3348459520, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.9.attn_qkv.weight", "offset": 3366481920, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.14.attn_norm.weight", "offset": 3393515520, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3393536000, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.14.ffn_up.weight", "offset": 3456614400, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.14.ffn_norm.weight", "offset": 3582771200, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3582791680, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.14.attn_qkv.weight", "offset": 3600814080, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.15.attn_norm.weight", "offset": 3627847680, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3627868160, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.15.ffn_up.weight", "offset": 3690946560, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.15.ffn_norm.weight", "offset": 3817103360, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3817123840, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.15.attn_qkv.weight", "offset": 3835146240, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.16.attn_norm.weight", "offset": 3862179840, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3862200320, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.16.ffn_up.weight", "offset": 3925278720, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.16.ffn_norm.weight", "offset": 4051435520, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4051456000, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.16.attn_qkv.weight", "offset": 4069478400, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.17.attn_norm.weight", "offset": 4096512000, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4096532480, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.17.ffn_up.weight", "offset": 4159610880, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.17.ffn_norm.weight", "offset": 4285767680, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 4285788160, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.17.attn_qkv.weight", "offset": 4303810560, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.18.attn_norm.weight", "offset": 4330844160, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4330864640, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.18.ffn_up.weight", "offset": 4393943040, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.18.ffn_norm.weight", "offset": 4520099840, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4520120320, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.18.attn_qkv.weight", "offset": 4538142720, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.19.attn_norm.weight", "offset": 4565176320, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4565196800, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.19.ffn_up.weight", "offset": 4628275200, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.19.ffn_norm.weight", "offset": 4754432000, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4754452480, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.19.attn_qkv.weight", "offset": 4772474880, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.20.attn_norm.weight", "offset": 4799508480, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4799528960, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.20.ffn_up.weight", "offset": 4862607360, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.20.ffn_norm.weight", "offset": 4988764160, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4988784640, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.20.attn_qkv.weight", "offset": 5006807040, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.21.attn_output.weight", "offset": 5033840640, "shape": [ 5120, 5120 ], "size": 27033600, "type": 6 }, { "name": "blk.21.attn_qkv.weight", "offset": 5051863040, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.21.attn_norm.weight", "offset": 5078896640, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5078917120, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.21.ffn_up.weight", "offset": 5141995520, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.21.ffn_norm.weight", "offset": 5268152320, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 5268172800, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5268193280, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.22.ffn_up.weight", "offset": 5331271680, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.22.ffn_norm.weight", "offset": 5457428480, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5457448960, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.22.attn_qkv.weight", "offset": 5475471360, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.23.attn_norm.weight", "offset": 5502504960, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5502525440, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.23.ffn_up.weight", "offset": 5565603840, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.23.ffn_norm.weight", "offset": 5691760640, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5691781120, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.23.attn_qkv.weight", "offset": 5709803520, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.24.attn_norm.weight", "offset": 5736837120, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5736857600, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.24.ffn_up.weight", "offset": 5799936000, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.24.ffn_norm.weight", "offset": 5926092800, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5926113280, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.24.attn_qkv.weight", "offset": 5944135680, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.25.attn_norm.weight", "offset": 5971169280, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5971189760, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.25.ffn_up.weight", "offset": 6034268160, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.25.ffn_norm.weight", "offset": 6160424960, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6160445440, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.25.attn_qkv.weight", "offset": 6178467840, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.26.attn_norm.weight", "offset": 6205501440, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6205521920, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.26.ffn_up.weight", "offset": 6268600320, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.26.ffn_norm.weight", "offset": 6394757120, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6394777600, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.26.attn_qkv.weight", "offset": 6412800000, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.27.attn_norm.weight", "offset": 6439833600, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6439854080, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.27.ffn_up.weight", "offset": 6502932480, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.27.ffn_norm.weight", "offset": 6629089280, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6629109760, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.27.attn_qkv.weight", "offset": 6647132160, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.28.attn_output.weight", "offset": 6674165760, "shape": [ 5120, 5120 ], "size": 27033600, "type": 6 }, { "name": "blk.28.attn_qkv.weight", "offset": 6692188160, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.28.attn_norm.weight", "offset": 6719221760, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6719242240, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.28.ffn_up.weight", "offset": 6782320640, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.28.ffn_norm.weight", "offset": 6908477440, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 6908497920, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6908518400, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.29.ffn_up.weight", "offset": 6971596800, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.29.ffn_norm.weight", "offset": 7097753600, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7097774080, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.29.attn_qkv.weight", "offset": 7115796480, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.30.attn_norm.weight", "offset": 7142830080, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7142850560, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.30.ffn_up.weight", "offset": 7205928960, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.30.ffn_norm.weight", "offset": 7332085760, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7332106240, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.30.attn_qkv.weight", "offset": 7350128640, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.31.attn_norm.weight", "offset": 7377162240, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7377182720, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.31.ffn_up.weight", "offset": 7440261120, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.31.ffn_norm.weight", "offset": 7566417920, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7566438400, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.31.attn_qkv.weight", "offset": 7584460800, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.32.attn_norm.weight", "offset": 7611494400, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7611514880, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.32.ffn_up.weight", "offset": 7674593280, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.32.ffn_norm.weight", "offset": 7800750080, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 7800770560, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.32.attn_qkv.weight", "offset": 7818792960, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.33.attn_norm.weight", "offset": 7845826560, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 7845847040, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.33.ffn_up.weight", "offset": 7908925440, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.33.ffn_norm.weight", "offset": 8035082240, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 8035102720, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.33.attn_qkv.weight", "offset": 8053125120, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.34.attn_norm.weight", "offset": 8080158720, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8080179200, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.34.ffn_up.weight", "offset": 8143257600, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.34.ffn_norm.weight", "offset": 8269414400, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 8269434880, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.34.attn_qkv.weight", "offset": 8287457280, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.35.attn_output.weight", "offset": 8314490880, "shape": [ 5120, 5120 ], "size": 27033600, "type": 6 }, { "name": "blk.35.attn_qkv.weight", "offset": 8332513280, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "output.weight", "offset": 8359546880, "shape": [ 5120, 32064 ], "size": 27033600, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 8494215680, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 8494236160, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.35.ffn_up.weight", "offset": 8557314560, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.35.ffn_norm.weight", "offset": 8683471360, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 8683491840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 8683512320, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.36.ffn_up.weight", "offset": 8746590720, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.36.ffn_norm.weight", "offset": 8872747520, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 8872768000, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.36.attn_qkv.weight", "offset": 8890790400, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.37.attn_norm.weight", "offset": 8917824000, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 8917844480, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.37.ffn_up.weight", "offset": 8980922880, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.37.ffn_norm.weight", "offset": 9107079680, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 9107100160, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.37.attn_qkv.weight", "offset": 9125122560, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.38.attn_norm.weight", "offset": 9152156160, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9152176640, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.38.ffn_up.weight", "offset": 9215255040, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.38.ffn_norm.weight", "offset": 9341411840, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 9341432320, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.38.attn_qkv.weight", "offset": 9359454720, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "blk.39.attn_norm.weight", "offset": 9386488320, "shape": [ 5120 ], "size": 27033600, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 9386508800, "shape": [ 17920, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.39.ffn_up.weight", "offset": 9449587200, "shape": [ 5120, 35840 ], "size": 63078400, "type": 6 }, { "name": "blk.39.ffn_norm.weight", "offset": 9575744000, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 9575764480, "shape": [ 5120, 5120 ], "size": 20480, "type": 6 }, { "name": "blk.39.attn_qkv.weight", "offset": 9593786880, "shape": [ 5120, 7680 ], "size": 18022400, "type": 6 }, { "name": "output_norm.weight", "offset": 9620820480, "shape": [ 5120 ], "size": 27033600, "type": 0 } ], "version": 3 }