phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 17, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 112865280, "shape": [ 5120 ], "size": 112865280, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 112885760, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 188149760, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.0.ffn_norm.weight", "offset": 314306560, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 314327040, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.attn_qkv.weight", "offset": 332349440, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 364605440, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 364625920, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 439889920, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.1.ffn_norm.weight", "offset": 566046720, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 566067200, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.attn_qkv.weight", "offset": 584089600, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 616345600, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 616366080, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 691630080, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.2.ffn_norm.weight", "offset": 817786880, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 817807360, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.attn_qkv.weight", "offset": 835829760, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 868085760, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 868106240, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 943370240, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.3.ffn_norm.weight", "offset": 1069527040, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1069547520, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.attn_qkv.weight", "offset": 1087569920, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1119825920, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1119846400, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1195110400, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.4.ffn_norm.weight", "offset": 1321267200, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1321287680, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.attn_qkv.weight", "offset": 1339310080, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1371566080, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1371586560, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1434664960, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.5.ffn_norm.weight", "offset": 1560821760, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1560842240, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.attn_qkv.weight", "offset": 1578864640, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 1611120640, "shape": [ 5120, 35840 ], "size": 32256000, "type": 13 }, { "name": "blk.6.attn_output.weight", "offset": 1737277440, "shape": [ 5120, 5120 ], "size": 126156800, "type": 13 }, { "name": "blk.6.attn_qkv.weight", "offset": 1755299840, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 1787555840, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1787576320, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 1850654720, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.10.ffn_norm.weight", "offset": 1976811520, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1976832000, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.attn_qkv.weight", "offset": 1994854400, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 2027110400, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2027130880, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 2102394880, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.11.ffn_norm.weight", "offset": 2228551680, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2228572160, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.11.attn_qkv.weight", "offset": 2246594560, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 2278850560, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2278871040, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 2341949440, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.12.ffn_norm.weight", "offset": 2468106240, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2468126720, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.attn_qkv.weight", "offset": 2486149120, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 2518405120, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2518425600, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 2581504000, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.13.ffn_norm.weight", "offset": 2707660800, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2707681280, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.attn_qkv.weight", "offset": 2725703680, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 2757959680, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2757980160, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 2833244160, "shape": [ 5120 ], "size": 75264000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2833264640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2833285120, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 2896363520, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.7.ffn_norm.weight", "offset": 3022520320, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 3022540800, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.attn_qkv.weight", "offset": 3040563200, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 3072819200, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3072839680, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 3135918080, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.8.ffn_norm.weight", "offset": 3262074880, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3262095360, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.attn_qkv.weight", "offset": 3280117760, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 3312373760, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3312394240, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 3387658240, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.9.ffn_norm.weight", "offset": 3513815040, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3513835520, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.9.attn_qkv.weight", "offset": 3531857920, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 3564113920, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3564134400, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 3627212800, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.14.ffn_norm.weight", "offset": 3753369600, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3753390080, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.attn_qkv.weight", "offset": 3771412480, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 3803668480, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3803688960, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 3866767360, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.15.ffn_norm.weight", "offset": 3992924160, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3992944640, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.attn_qkv.weight", "offset": 4010967040, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 4043223040, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4043243520, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 4118507520, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.16.ffn_norm.weight", "offset": 4244664320, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4244684800, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.16.attn_qkv.weight", "offset": 4262707200, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 4294963200, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4294983680, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 4358062080, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.17.ffn_norm.weight", "offset": 4484218880, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 4484239360, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.attn_qkv.weight", "offset": 4502261760, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 4534517760, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4534538240, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 4597616640, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.18.ffn_norm.weight", "offset": 4723773440, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4723793920, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.attn_qkv.weight", "offset": 4741816320, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 4774072320, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4774092800, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 4849356800, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.19.ffn_norm.weight", "offset": 4975513600, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4975534080, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.19.attn_qkv.weight", "offset": 4993556480, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 5025812480, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5025832960, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 5088911360, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.20.ffn_norm.weight", "offset": 5215068160, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 5215088640, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.attn_qkv.weight", "offset": 5233111040, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 5265367040, "shape": [ 5120, 5120 ], "size": 32256000, "type": 13 }, { "name": "blk.21.attn_qkv.weight", "offset": 5283389440, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 5315645440, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5315665920, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 5378744320, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.21.ffn_norm.weight", "offset": 5504901120, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 5504921600, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5504942080, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 5580206080, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.22.ffn_norm.weight", "offset": 5706362880, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5706383360, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.22.attn_qkv.weight", "offset": 5724405760, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 5756661760, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5756682240, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 5819760640, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.23.ffn_norm.weight", "offset": 5945917440, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5945937920, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.attn_qkv.weight", "offset": 5963960320, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 5996216320, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5996236800, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 6059315200, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.24.ffn_norm.weight", "offset": 6185472000, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 6185492480, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.attn_qkv.weight", "offset": 6203514880, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 6235770880, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6235791360, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 6311055360, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.25.ffn_norm.weight", "offset": 6437212160, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6437232640, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.25.attn_qkv.weight", "offset": 6455255040, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 6487511040, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6487531520, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 6550609920, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.26.ffn_norm.weight", "offset": 6676766720, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6676787200, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.attn_qkv.weight", "offset": 6694809600, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 6727065600, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6727086080, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 6790164480, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.27.ffn_norm.weight", "offset": 6916321280, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6916341760, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.attn_qkv.weight", "offset": 6934364160, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 6966620160, "shape": [ 5120, 5120 ], "size": 32256000, "type": 13 }, { "name": "blk.28.attn_qkv.weight", "offset": 6984642560, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 7016898560, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 7016919040, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 7092183040, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.28.ffn_norm.weight", "offset": 7218339840, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 7218360320, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 7218380800, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 7281459200, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.29.ffn_norm.weight", "offset": 7407616000, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7407636480, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.attn_qkv.weight", "offset": 7425658880, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 7457914880, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7457935360, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 7521013760, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.30.ffn_norm.weight", "offset": 7647170560, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7647191040, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.attn_qkv.weight", "offset": 7665213440, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 7697469440, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7697489920, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "\ufffdhl\ufffd\u000bk1Z\u001c\ufffdK\u0007\ufffdܙ^Y@\ufffd\u0006", "offset": 7772753920, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.31.ffn_norm.weight", "offset": 7898910720, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7898931200, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.31.attn_qkv.weight", "offset": 7916953600, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 7949209600, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 7949230080, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 8012308480, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.32.ffn_norm.weight", "offset": 8138465280, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 8138485760, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.attn_qkv.weight", "offset": 8156508160, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 8188764160, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 8188784640, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 8251863040, "shape": [ 5120, 35840 ], "size": 63078400, "type": 13 }, { "name": "blk.33.ffn_norm.weight", "offset": 8378019840, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 8378040320, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.attn_qkv.weight", "offset": 8396062720, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 8428318720, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8428339200, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 8503603200, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.34.ffn_norm.weight", "offset": 8629760000, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 8629780480, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.34.attn_qkv.weight", "offset": 8647802880, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 8680058880, "shape": [ 5120, 5120 ], "size": 32256000, "type": 13 }, { "name": "blk.35.attn_qkv.weight", "offset": 8698081280, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "output.weight", "offset": 8730337280, "shape": [ 5120, 32064 ], "size": 32256000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 8865006080, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 8865026560, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 8940290560, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.35.ffn_norm.weight", "offset": 9066447360, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 9066467840, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 9066488320, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 9141752320, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.36.ffn_norm.weight", "offset": 9267909120, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 9267929600, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.36.attn_qkv.weight", "offset": 9285952000, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 9318208000, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 9318228480, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 9393492480, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.37.ffn_norm.weight", "offset": 9519649280, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 9519669760, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.37.attn_qkv.weight", "offset": 9537692160, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 9569948160, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9569968640, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 9645232640, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.38.ffn_norm.weight", "offset": 9771389440, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 9771409920, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.38.attn_qkv.weight", "offset": 9789432320, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 9821688320, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 9821708800, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 9896972800, "shape": [ 5120, 35840 ], "size": 75264000, "type": 13 }, { "name": "blk.39.ffn_norm.weight", "offset": 10023129600, "shape": [ 5120 ], "size": 126156800, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 10023150080, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.39.attn_qkv.weight", "offset": 10041172480, "shape": [ 5120, 7680 ], "size": 18022400, "type": 14 }, { "name": "output_norm.weight", "offset": 10073428480, "shape": [ 5120 ], "size": 32256000, "type": 0 } ], "version": 3 }