phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 14, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 92344320, "shape": [ 5120 ], "size": 92344320, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 92364800, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 155443200, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.0.ffn_norm.weight", "offset": 258662400, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 258682880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.0.attn_qkv.weight", "offset": 273428480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 295546880, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 295567360, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 358645760, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.1.ffn_norm.weight", "offset": 461864960, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 461885440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.1.attn_qkv.weight", "offset": 476631040, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 498749440, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 498769920, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 561848320, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.2.ffn_norm.weight", "offset": 665067520, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 665088000, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.attn_qkv.weight", "offset": 679833600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 701952000, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 701972480, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 765050880, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.3.ffn_norm.weight", "offset": 868270080, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 868290560, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.attn_qkv.weight", "offset": 883036160, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 905154560, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 905175040, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 968253440, "shape": [ 5120, 35840 ], "size": 63078400, "type": 12 }, { "name": "blk.4.ffn_norm.weight", "offset": 1071472640, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1071493120, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.attn_qkv.weight", "offset": 1086238720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 1108357120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1108377600, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 1159987200, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.5.ffn_norm.weight", "offset": 1263206400, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1263226880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.attn_qkv.weight", "offset": 1277972480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1300090880, "shape": [ 5120, 35840 ], "size": 22118400, "type": 12 }, { "name": "blk.6.attn_output.weight", "offset": 1403310080, "shape": [ 5120, 5120 ], "size": 103219200, "type": 12 }, { "name": "blk.6.attn_qkv.weight", "offset": 1418055680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1440174080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1440194560, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1491804160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.10.ffn_norm.weight", "offset": 1595023360, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1595043840, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.attn_qkv.weight", "offset": 1609789440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1631907840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1631928320, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1683537920, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.11.ffn_norm.weight", "offset": 1786757120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1786777600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.attn_qkv.weight", "offset": 1801523200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1823641600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1823662080, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1875271680, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.12.ffn_norm.weight", "offset": 1978490880, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1978511360, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.attn_qkv.weight", "offset": 1993256960, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 2015375360, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2015395840, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 2067005440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.13.ffn_norm.weight", "offset": 2170224640, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2170245120, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.attn_qkv.weight", "offset": 2184990720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 2207109120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2207129600, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 2258739200, "shape": [ 5120 ], "size": 51609600, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2258759680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2258780160, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 2310389760, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.7.ffn_norm.weight", "offset": 2413608960, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2413629440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.attn_qkv.weight", "offset": 2428375040, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2450493440, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2450513920, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2502123520, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.8.ffn_norm.weight", "offset": 2605342720, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2605363200, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.attn_qkv.weight", "offset": 2620108800, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2642227200, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2642247680, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2693857280, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.9.ffn_norm.weight", "offset": 2797076480, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2797096960, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.attn_qkv.weight", "offset": 2811842560, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2833960960, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2833981440, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 2885591040, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.14.ffn_norm.weight", "offset": 2988810240, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2988830720, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.attn_qkv.weight", "offset": 3003576320, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 3025694720, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3025715200, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 3077324800, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.15.ffn_norm.weight", "offset": 3180544000, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3180564480, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.attn_qkv.weight", "offset": 3195310080, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 3217428480, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3217448960, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 3269058560, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.16.ffn_norm.weight", "offset": 3372277760, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3372298240, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.attn_qkv.weight", "offset": 3387043840, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 3409162240, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3409182720, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 3460792320, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.17.ffn_norm.weight", "offset": 3564011520, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3564032000, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.attn_qkv.weight", "offset": 3578777600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3600896000, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3600916480, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3652526080, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.18.ffn_norm.weight", "offset": 3755745280, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3755765760, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.attn_qkv.weight", "offset": 3770511360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 3792629760, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3792650240, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 3844259840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.19.ffn_norm.weight", "offset": 3947479040, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3947499520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.attn_qkv.weight", "offset": 3962245120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3984363520, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3984384000, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 4035993600, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.20.ffn_norm.weight", "offset": 4139212800, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4139233280, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.attn_qkv.weight", "offset": 4153978880, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 4176097280, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.21.attn_qkv.weight", "offset": 4190842880, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 4212961280, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4212981760, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 4264591360, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.21.ffn_norm.weight", "offset": 4367810560, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4367831040, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4367851520, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 4419461120, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.22.ffn_norm.weight", "offset": 4522680320, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4522700800, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.attn_qkv.weight", "offset": 4537446400, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 4559564800, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4559585280, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 4611194880, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.23.ffn_norm.weight", "offset": 4714414080, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4714434560, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.attn_qkv.weight", "offset": 4729180160, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4751298560, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4751319040, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 4802928640, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.24.ffn_norm.weight", "offset": 4906147840, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4906168320, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.attn_qkv.weight", "offset": 4920913920, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4943032320, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4943052800, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 4994662400, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.25.ffn_norm.weight", "offset": 5097881600, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5097902080, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.attn_qkv.weight", "offset": 5112647680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 5134766080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5134786560, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 5186396160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.26.ffn_norm.weight", "offset": 5289615360, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5289635840, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.attn_qkv.weight", "offset": 5304381440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 5326499840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5326520320, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 5378129920, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.27.ffn_norm.weight", "offset": 5481349120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5481369600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.attn_qkv.weight", "offset": 5496115200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 5518233600, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.28.attn_qkv.weight", "offset": 5532979200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 5555097600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5555118080, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 5606727680, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.28.ffn_norm.weight", "offset": 5709946880, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 5709967360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5709987840, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 5761597440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.29.ffn_norm.weight", "offset": 5864816640, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5864837120, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.attn_qkv.weight", "offset": 5879582720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 5901701120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5901721600, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 5953331200, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.30.ffn_norm.weight", "offset": 6056550400, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 6056570880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.attn_qkv.weight", "offset": 6071316480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 6093434880, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6093455360, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 6145064960, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.31.ffn_norm.weight", "offset": 6248284160, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6248304640, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.attn_qkv.weight", "offset": 6263050240, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 6285168640, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6285189120, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 6336798720, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.32.ffn_norm.weight", "offset": 6440017920, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6440038400, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.attn_qkv.weight", "offset": 6454784000, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 6476902400, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6476922880, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 6528532480, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.33.ffn_norm.weight", "offset": 6631751680, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 6631772160, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.attn_qkv.weight", "offset": 6646517760, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 6668636160, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6668656640, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 6720266240, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.34.ffn_norm.weight", "offset": 6823485440, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 6823505920, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.attn_qkv.weight", "offset": 6838251520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 6860369920, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.35.attn_qkv.weight", "offset": 6875115520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output.weight", "offset": 6897233920, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 7031902720, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7031923200, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 7083532800, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.35.ffn_norm.weight", "offset": 7186752000, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7186772480, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7186792960, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 7238402560, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.36.ffn_norm.weight", "offset": 7341621760, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 7341642240, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.attn_qkv.weight", "offset": 7356387840, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 7378506240, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7378526720, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 7430136320, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.37.ffn_norm.weight", "offset": 7533355520, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 7533376000, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.attn_qkv.weight", "offset": 7548121600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 7570240000, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7570260480, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 7621870080, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.38.ffn_norm.weight", "offset": 7725089280, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 7725109760, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.attn_qkv.weight", "offset": 7739855360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 7761973760, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7761994240, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 7813603840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 12 }, { "name": "blk.39.ffn_norm.weight", "offset": 7916823040, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 7916843520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.attn_qkv.weight", "offset": 7931589120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output_norm.weight", "offset": 7953707520, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }