phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 9, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 123125760, "shape": [ 5120 ], "size": 123125760, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 123146240, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.0.ffn_up.weight", "offset": 191959040, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.0.ffn_norm.weight", "offset": 329584640, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 329605120, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.0.attn_qkv.weight", "offset": 349265920, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.1.attn_norm.weight", "offset": 378757120, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 378777600, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.1.ffn_up.weight", "offset": 447590400, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.1.ffn_norm.weight", "offset": 585216000, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 585236480, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.1.attn_qkv.weight", "offset": 604897280, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.2.attn_norm.weight", "offset": 634388480, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 634408960, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.2.ffn_up.weight", "offset": 703221760, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.2.ffn_norm.weight", "offset": 840847360, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 840867840, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.2.attn_qkv.weight", "offset": 860528640, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.3.attn_norm.weight", "offset": 890019840, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 890040320, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.3.ffn_up.weight", "offset": 958853120, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.3.ffn_norm.weight", "offset": 1096478720, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1096499200, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.3.attn_qkv.weight", "offset": 1116160000, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.4.attn_norm.weight", "offset": 1145651200, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1145671680, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.4.ffn_up.weight", "offset": 1214484480, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.4.ffn_norm.weight", "offset": 1352110080, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1352130560, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.4.attn_qkv.weight", "offset": 1371791360, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.5.attn_norm.weight", "offset": 1401282560, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1401303040, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.5.ffn_up.weight", "offset": 1470115840, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.5.ffn_norm.weight", "offset": 1607741440, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1607761920, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.5.attn_qkv.weight", "offset": 1627422720, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.6.ffn_up.weight", "offset": 1656913920, "shape": [ 5120, 35840 ], "size": 29491200, "type": 7 }, { "name": "blk.6.attn_output.weight", "offset": 1794539520, "shape": [ 5120, 5120 ], "size": 137625600, "type": 7 }, { "name": "blk.6.attn_qkv.weight", "offset": 1814200320, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.10.attn_norm.weight", "offset": 1843691520, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1843712000, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.10.ffn_up.weight", "offset": 1912524800, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.10.ffn_norm.weight", "offset": 2050150400, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 2050170880, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.10.attn_qkv.weight", "offset": 2069831680, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.11.attn_norm.weight", "offset": 2099322880, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2099343360, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.11.ffn_up.weight", "offset": 2168156160, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.11.ffn_norm.weight", "offset": 2305781760, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2305802240, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.11.attn_qkv.weight", "offset": 2325463040, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.12.attn_norm.weight", "offset": 2354954240, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2354974720, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.12.ffn_up.weight", "offset": 2423787520, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.12.ffn_norm.weight", "offset": 2561413120, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2561433600, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.12.attn_qkv.weight", "offset": 2581094400, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.13.attn_norm.weight", "offset": 2610585600, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2610606080, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.13.ffn_up.weight", "offset": 2679418880, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.13.ffn_norm.weight", "offset": 2817044480, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2817064960, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.13.attn_qkv.weight", "offset": 2836725760, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.6.attn_norm.weight", "offset": 2866216960, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2866237440, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.6.ffn_norm.weight", "offset": 2935050240, "shape": [ 5120 ], "size": 68812800, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2935070720, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2935091200, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.7.ffn_up.weight", "offset": 3003904000, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.7.ffn_norm.weight", "offset": 3141529600, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 3141550080, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.7.attn_qkv.weight", "offset": 3161210880, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.8.attn_norm.weight", "offset": 3190702080, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3190722560, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.8.ffn_up.weight", "offset": 3259535360, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.8.ffn_norm.weight", "offset": 3397160960, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3397181440, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.8.attn_qkv.weight", "offset": 3416842240, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.9.attn_norm.weight", "offset": 3446333440, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3446353920, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.9.ffn_up.weight", "offset": 3515166720, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.9.ffn_norm.weight", "offset": 3652792320, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3652812800, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.9.attn_qkv.weight", "offset": 3672473600, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.14.attn_norm.weight", "offset": 3701964800, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3701985280, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.14.ffn_up.weight", "offset": 3770798080, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.14.ffn_norm.weight", "offset": 3908423680, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3908444160, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.14.attn_qkv.weight", "offset": 3928104960, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.15.attn_norm.weight", "offset": 3957596160, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3957616640, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.15.ffn_up.weight", "offset": 4026429440, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.15.ffn_norm.weight", "offset": 4164055040, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 4164075520, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.15.attn_qkv.weight", "offset": 4183736320, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.16.attn_norm.weight", "offset": 4213227520, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4213248000, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.16.ffn_up.weight", "offset": 4282060800, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.16.ffn_norm.weight", "offset": 4419686400, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4419706880, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.16.attn_qkv.weight", "offset": 4439367680, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.17.attn_norm.weight", "offset": 4468858880, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4468879360, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.17.ffn_up.weight", "offset": 4537692160, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.17.ffn_norm.weight", "offset": 4675317760, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 4675338240, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.17.attn_qkv.weight", "offset": 4694999040, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.18.attn_norm.weight", "offset": 4724490240, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 4724510720, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.18.ffn_up.weight", "offset": 4793323520, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.18.ffn_norm.weight", "offset": 4930949120, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4930969600, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.18.attn_qkv.weight", "offset": 4950630400, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.19.attn_norm.weight", "offset": 4980121600, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4980142080, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.19.ffn_up.weight", "offset": 5048954880, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.19.ffn_norm.weight", "offset": 5186580480, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 5186600960, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.19.attn_qkv.weight", "offset": 5206261760, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.20.attn_norm.weight", "offset": 5235752960, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5235773440, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.20.ffn_up.weight", "offset": 5304586240, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.20.ffn_norm.weight", "offset": 5442211840, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 5442232320, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.20.attn_qkv.weight", "offset": 5461893120, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.21.attn_output.weight", "offset": 5491384320, "shape": [ 5120, 5120 ], "size": 29491200, "type": 7 }, { "name": "blk.21.attn_qkv.weight", "offset": 5511045120, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.21.attn_norm.weight", "offset": 5540536320, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5540556800, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.21.ffn_up.weight", "offset": 5609369600, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.21.ffn_norm.weight", "offset": 5746995200, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 5747015680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5747036160, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.22.ffn_up.weight", "offset": 5815848960, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.22.ffn_norm.weight", "offset": 5953474560, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5953495040, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.22.attn_qkv.weight", "offset": 5973155840, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.23.attn_norm.weight", "offset": 6002647040, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6002667520, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.23.ffn_up.weight", "offset": 6071480320, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.23.ffn_norm.weight", "offset": 6209105920, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 6209126400, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.23.attn_qkv.weight", "offset": 6228787200, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.24.attn_norm.weight", "offset": 6258278400, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 6258298880, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.24.ffn_up.weight", "offset": 6327111680, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.24.ffn_norm.weight", "offset": 6464737280, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 6464757760, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.24.attn_qkv.weight", "offset": 6484418560, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.25.attn_norm.weight", "offset": 6513909760, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6513930240, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.25.ffn_up.weight", "offset": 6582743040, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.25.ffn_norm.weight", "offset": 6720368640, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6720389120, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.25.attn_qkv.weight", "offset": 6740049920, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.26.attn_norm.weight", "offset": 6769541120, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6769561600, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.26.ffn_up.weight", "offset": 6838374400, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.26.ffn_norm.weight", "offset": 6976000000, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6976020480, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.26.attn_qkv.weight", "offset": 6995681280, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.27.attn_norm.weight", "offset": 7025172480, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 7025192960, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.27.ffn_up.weight", "offset": 7094005760, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.27.ffn_norm.weight", "offset": 7231631360, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 7231651840, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.27.attn_qkv.weight", "offset": 7251312640, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.28.attn_output.weight", "offset": 7280803840, "shape": [ 5120, 5120 ], "size": 29491200, "type": 7 }, { "name": "blk.28.attn_qkv.weight", "offset": 7300464640, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.28.attn_norm.weight", "offset": 7329955840, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 7329976320, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.28.ffn_up.weight", "offset": 7398789120, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.28.ffn_norm.weight", "offset": 7536414720, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 7536435200, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 7536455680, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.29.ffn_up.weight", "offset": 7605268480, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.29.ffn_norm.weight", "offset": 7742894080, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7742914560, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.29.attn_qkv.weight", "offset": 7762575360, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.30.attn_norm.weight", "offset": 7792066560, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7792087040, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.30.ffn_up.weight", "offset": 7860899840, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.30.ffn_norm.weight", "offset": 7998525440, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7998545920, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.30.attn_qkv.weight", "offset": 8018206720, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.31.attn_norm.weight", "offset": 8047697920, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 8047718400, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.31.ffn_up.weight", "offset": 8116531200, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.31.ffn_norm.weight", "offset": 8254156800, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 8254177280, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.31.attn_qkv.weight", "offset": 8273838080, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.32.attn_norm.weight", "offset": 8303329280, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 8303349760, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.32.ffn_up.weight", "offset": 8372162560, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.32.ffn_norm.weight", "offset": 8509788160, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 8509808640, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.32.attn_qkv.weight", "offset": 8529469440, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.33.attn_norm.weight", "offset": 8558960640, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 8558981120, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.33.ffn_up.weight", "offset": 8627793920, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.33.ffn_norm.weight", "offset": 8765419520, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 8765440000, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.33.attn_qkv.weight", "offset": 8785100800, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.34.attn_norm.weight", "offset": 8814592000, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 8814612480, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.34.ffn_up.weight", "offset": 8883425280, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.34.ffn_norm.weight", "offset": 9021050880, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 9021071360, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.34.attn_qkv.weight", "offset": 9040732160, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.35.attn_output.weight", "offset": 9070223360, "shape": [ 5120, 5120 ], "size": 29491200, "type": 7 }, { "name": "blk.35.attn_qkv.weight", "offset": 9089884160, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "output.weight", "offset": 9119375360, "shape": [ 5120, 32064 ], "size": 29491200, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 9254044160, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 9254064640, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.35.ffn_up.weight", "offset": 9322877440, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.35.ffn_norm.weight", "offset": 9460503040, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 9460523520, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 9460544000, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.36.ffn_up.weight", "offset": 9529356800, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.36.ffn_norm.weight", "offset": 9666982400, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 9667002880, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.36.attn_qkv.weight", "offset": 9686663680, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.37.attn_norm.weight", "offset": 9716154880, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 9716175360, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.37.ffn_up.weight", "offset": 9784988160, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.37.ffn_norm.weight", "offset": 9922613760, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 9922634240, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.37.attn_qkv.weight", "offset": 9942295040, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.38.attn_norm.weight", "offset": 9971786240, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 9971806720, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.38.ffn_up.weight", "offset": 10040619520, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.38.ffn_norm.weight", "offset": 10178245120, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 10178265600, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.38.attn_qkv.weight", "offset": 10197926400, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "blk.39.attn_norm.weight", "offset": 10227417600, "shape": [ 5120 ], "size": 29491200, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 10227438080, "shape": [ 17920, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.39.ffn_up.weight", "offset": 10296250880, "shape": [ 5120, 35840 ], "size": 68812800, "type": 7 }, { "name": "blk.39.ffn_norm.weight", "offset": 10433876480, "shape": [ 5120 ], "size": 137625600, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 10433896960, "shape": [ 5120, 5120 ], "size": 20480, "type": 7 }, { "name": "blk.39.attn_qkv.weight", "offset": 10453557760, "shape": [ 5120, 7680 ], "size": 19660800, "type": 7 }, { "name": "output_norm.weight", "offset": 10483048960, "shape": [ 5120 ], "size": 29491200, "type": 0 } ], "version": 3 }