phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 14, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 12 }, { "name": "blk.0.attn_norm.weight", "offset": 92344832, "shape": [ 5120 ], "size": 92344320, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 92365312, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 155443712, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 258662912, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 258683392, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.0.attn_qkv.weight", "offset": 273428992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 295547392, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 295567872, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 358646272, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 461865472, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 461885952, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.1.attn_qkv.weight", "offset": 476631552, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 498749952, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 498770432, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 561848832, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 665068032, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 665088512, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.attn_qkv.weight", "offset": 679834112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 701952512, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 701972992, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 765051392, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 868270592, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 868291072, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.attn_qkv.weight", "offset": 883036672, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 905155072, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 905175552, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 968253952, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1071473152, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1071493632, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.attn_qkv.weight", "offset": 1086239232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1108357632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1108378112, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1159987712, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1263206912, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1263227392, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.attn_qkv.weight", "offset": 1277972992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1300091392, "shape": [ 5120, 35840 ], "size": 22118400, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 1403310592, "shape": [ 5120, 5120 ], "size": 103219200, "type": 12 }, { "name": "blk.6.attn_qkv.weight", "offset": 1418056192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1440174592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1440195072, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1491804672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1595023872, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1595044352, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.attn_qkv.weight", "offset": 1609789952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1631908352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1631928832, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1683538432, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1786757632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1786778112, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.attn_qkv.weight", "offset": 1801523712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1823642112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1823662592, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1875272192, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1978491392, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1978511872, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.attn_qkv.weight", "offset": 1993257472, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 2015375872, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2015396352, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 2067005952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 2170225152, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2170245632, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.attn_qkv.weight", "offset": 2184991232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 2207109632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2207130112, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 2258739712, "shape": [ 5120 ], "size": 51609600, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2258760192, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2258780672, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2310390272, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2413609472, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2413629952, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.attn_qkv.weight", "offset": 2428375552, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2450493952, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2450514432, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2502124032, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2605343232, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2605363712, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.attn_qkv.weight", "offset": 2620109312, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2642227712, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2642248192, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2693857792, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2797076992, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2797097472, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.attn_qkv.weight", "offset": 2811843072, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2833961472, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2833981952, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 2885591552, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 2988810752, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2988831232, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.attn_qkv.weight", "offset": 3003576832, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 3025695232, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3025715712, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 3077325312, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 3180544512, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3180564992, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.attn_qkv.weight", "offset": 3195310592, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 3217428992, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3217449472, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 3269059072, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 3372278272, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3372298752, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.attn_qkv.weight", "offset": 3387044352, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 3409162752, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3409183232, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 3460792832, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 3564012032, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3564032512, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.attn_qkv.weight", "offset": 3578778112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3600896512, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3600916992, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3652526592, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 3755745792, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3755766272, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.attn_qkv.weight", "offset": 3770511872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 3792630272, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3792650752, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 3844260352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "output.weight\u0002\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000", "offset": 3947479552, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3947500032, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.attn_qkv.weight", "offset": 3962245632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3984364032, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3984384512, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 4035994112, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 4139213312, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4139233792, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.attn_qkv.weight", "offset": 4153979392, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 4176097792, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.21.attn_qkv.weight", "offset": 4190843392, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 4212961792, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4212982272, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 4264591872, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 4367811072, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4367831552, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4367852032, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 4419461632, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 4522680832, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4522701312, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.attn_qkv.weight", "offset": 4537446912, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 4559565312, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4559585792, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 4611195392, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 4714414592, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4714435072, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.attn_qkv.weight", "offset": 4729180672, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4751299072, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4751319552, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 4802929152, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 4906148352, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4906168832, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.attn_qkv.weight", "offset": 4920914432, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4943032832, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4943053312, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 4994662912, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 5097882112, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5097902592, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.attn_qkv.weight", "offset": 5112648192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 5134766592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5134787072, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 5186396672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 5289615872, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5289636352, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.attn_qkv.weight", "offset": 5304381952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 5326500352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5326520832, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 5378130432, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 5481349632, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5481370112, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.attn_qkv.weight", "offset": 5496115712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 5518234112, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.28.attn_qkv.weight", "offset": 5532979712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 5555098112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5555118592, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 5606728192, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 5709947392, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 5709967872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5709988352, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 5761597952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 5864817152, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5864837632, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.attn_qkv.weight", "offset": 5879583232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 5901701632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5901722112, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 5953331712, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 6056550912, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 6056571392, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.attn_qkv.weight", "offset": 6071316992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 6093435392, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6093455872, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 6145065472, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 6248284672, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6248305152, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.attn_qkv.weight", "offset": 6263050752, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 6285169152, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6285189632, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 6336799232, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 6440018432, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6440038912, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.attn_qkv.weight", "offset": 6454784512, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 6476902912, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6476923392, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 6528532992, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 6631752192, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 6631772672, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.attn_qkv.weight", "offset": 6646518272, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 6668636672, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6668657152, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 6720266752, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 6823485952, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 6823506432, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.attn_qkv.weight", "offset": 6838252032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 6860370432, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.35.attn_qkv.weight", "offset": 6875116032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output.weight", "offset": 6897234432, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 7031903232, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7031923712, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 7083533312, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 7186752512, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7186772992, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7186793472, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 7238403072, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 7341622272, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 7341642752, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.attn_qkv.weight", "offset": 7356388352, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 7378506752, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7378527232, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 7430136832, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 7533356032, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 7533376512, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.attn_qkv.weight", "offset": 7548122112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 7570240512, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7570260992, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 7621870592, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 7725089792, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 7725110272, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.attn_qkv.weight", "offset": 7739855872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 7761974272, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7761994752, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 7813604352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 7916823552, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 7916844032, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.attn_qkv.weight", "offset": 7931589632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output_norm.weight", "offset": 7953708032, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }