phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 13, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 70540800, "shape": [ 5120 ], "size": 70540800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 70561280, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 133639680, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 212487680, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 212508160, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.attn_qkv.weight", "offset": 230530560, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 252648960, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 252669440, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 315747840, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 394595840, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 394616320, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.attn_qkv.weight", "offset": 412638720, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 434757120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 434777600, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.ffn_up.weight", "offset": 497856000, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 576704000, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 576724480, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.2.attn_qkv.weight", "offset": 594746880, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 616865280, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 616885760, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.ffn_up.weight", "offset": 679964160, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 758812160, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 758832640, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.3.attn_qkv.weight", "offset": 776855040, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 798973440, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 798993920, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.ffn_up.weight", "offset": 862072320, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 940920320, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 940940800, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.4.attn_qkv.weight", "offset": 958963200, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 981081600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 981102080, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.ffn_up.weight", "offset": 1044180480, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1123028480, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1123048960, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.5.attn_qkv.weight", "offset": 1141071360, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1163189760, "shape": [ 5120, 35840 ], "size": 22118400, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1242037760, "shape": [ 5120, 5120 ], "size": 78848000, "type": 13 }, { "name": "blk.6.attn_qkv.weight", "offset": 1260060160, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1282178560, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1282199040, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.ffn_up.weight", "offset": 1345277440, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1424125440, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1424145920, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.10.attn_qkv.weight", "offset": 1442168320, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1464286720, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1464307200, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.11.ffn_up.weight", "offset": 1527385600, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1606233600, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1606254080, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.11.attn_qkv.weight", "offset": 1624276480, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1646394880, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1646415360, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.ffn_up.weight", "offset": 1709493760, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1788341760, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1788362240, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.12.attn_qkv.weight", "offset": 1806384640, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1828503040, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1828523520, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.ffn_up.weight", "offset": 1891601920, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1970449920, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1970470400, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.13.attn_qkv.weight", "offset": 1988492800, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 2010611200, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2010631680, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.6.ffn_norm.weight", "offset": 2073710080, "shape": [ 5120 ], "size": 63078400, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2073730560, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2073751040, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.ffn_up.weight", "offset": 2136829440, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 2215677440, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2215697920, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.7.attn_qkv.weight", "offset": 2233720320, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2255838720, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2255859200, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.ffn_up.weight", "offset": 2318937600, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 2397785600, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2397806080, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.8.attn_qkv.weight", "offset": 2415828480, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2437946880, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2437967360, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.9.ffn_up.weight", "offset": 2501045760, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2579893760, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2579914240, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.9.attn_qkv.weight", "offset": 2597936640, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2620055040, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2620075520, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.ffn_up.weight", "offset": 2683153920, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2762001920, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2762022400, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.14.attn_qkv.weight", "offset": 2780044800, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 2802163200, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2802183680, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.ffn_up.weight", "offset": 2865262080, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2944110080, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2944130560, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.15.attn_qkv.weight", "offset": 2962152960, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2984271360, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2984291840, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.16.ffn_up.weight", "offset": 3047370240, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 3126218240, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3126238720, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.16.attn_qkv.weight", "offset": 3144261120, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 3166379520, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3166400000, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.ffn_up.weight", "offset": 3229478400, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 3308326400, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3308346880, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.17.attn_qkv.weight", "offset": 3326369280, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3348487680, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3348508160, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.ffn_up.weight", "offset": 3411586560, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 3490434560, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3490455040, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.18.attn_qkv.weight", "offset": 3508477440, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 3530595840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3530616320, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.19.ffn_up.weight", "offset": 3593694720, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 3672542720, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3672563200, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.19.attn_qkv.weight", "offset": 3690585600, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3712704000, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3712724480, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.ffn_up.weight", "offset": 3775802880, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 3854650880, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3854671360, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.20.attn_qkv.weight", "offset": 3872693760, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 3894812160, "shape": [ 5120, 5120 ], "size": 22118400, "type": 13 }, { "name": "blk.21.attn_qkv.weight", "offset": 3912834560, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 3934952960, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3934973440, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.21.ffn_up.weight", "offset": 3998051840, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 4076899840, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4076920320, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4076940800, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.22.ffn_up.weight", "offset": 4140019200, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 4218867200, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4218887680, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.22.attn_qkv.weight", "offset": 4236910080, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 4259028480, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4259048960, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.ffn_up.weight", "offset": 4322127360, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 4400975360, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4400995840, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.23.attn_qkv.weight", "offset": 4419018240, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4441136640, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4441157120, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.ffn_up.weight", "offset": 4504235520, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 4583083520, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4583104000, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.24.attn_qkv.weight", "offset": 4601126400, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4623244800, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4623265280, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.25.ffn_up.weight", "offset": 4686343680, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 4765191680, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 4765212160, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.25.attn_qkv.weight", "offset": 4783234560, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 4805352960, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4805373440, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.ffn_up.weight", "offset": 4868451840, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 4947299840, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 4947320320, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.26.attn_qkv.weight", "offset": 4965342720, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 4987461120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4987481600, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.ffn_up.weight", "offset": 5050560000, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 5129408000, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5129428480, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.27.attn_qkv.weight", "offset": 5147450880, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 5169569280, "shape": [ 5120, 5120 ], "size": 22118400, "type": 13 }, { "name": "blk.28.attn_qkv.weight", "offset": 5187591680, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 5209710080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5209730560, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.28.ffn_up.weight", "offset": 5272808960, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 5351656960, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 5351677440, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5351697920, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.ffn_up.weight", "offset": 5414776320, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 5493624320, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5493644800, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.29.attn_qkv.weight", "offset": 5511667200, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 5533785600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5533806080, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.ffn_up.weight", "offset": 5596884480, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 5675732480, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 5675752960, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.30.attn_qkv.weight", "offset": 5693775360, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 5715893760, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5715914240, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.31.ffn_up.weight", "offset": 5778992640, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 5857840640, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 5857861120, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.31.attn_qkv.weight", "offset": 5875883520, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 5898001920, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 5898022400, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.ffn_up.weight", "offset": 5961100800, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 6039948800, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6039969280, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.32.attn_qkv.weight", "offset": 6057991680, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 6080110080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6080130560, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.ffn_up.weight", "offset": 6143208960, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 6222056960, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 6222077440, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.33.attn_qkv.weight", "offset": 6240099840, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 6262218240, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6262238720, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.34.ffn_up.weight", "offset": 6325317120, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 6404165120, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 6404185600, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.34.attn_qkv.weight", "offset": 6422208000, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 6444326400, "shape": [ 5120, 5120 ], "size": 22118400, "type": 13 }, { "name": "blk.35.attn_qkv.weight", "offset": 6462348800, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "output.weight", "offset": 6484467200, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 6619136000, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6619156480, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.35.ffn_up.weight", "offset": 6682234880, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 6761082880, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 6761103360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6761123840, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.36.ffn_up.weight", "offset": 6824202240, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 6903050240, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 6903070720, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.36.attn_qkv.weight", "offset": 6921093120, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 6943211520, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 6943232000, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.37.ffn_up.weight", "offset": 7006310400, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 7085158400, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 7085178880, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.37.attn_qkv.weight", "offset": 7103201280, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 7125319680, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7125340160, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.38.ffn_up.weight", "offset": 7188418560, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 7267266560, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 7267287040, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.38.attn_qkv.weight", "offset": 7285309440, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 7307427840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7307448320, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.39.ffn_up.weight", "offset": 7370526720, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 7449374720, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 7449395200, "shape": [ 5120, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.39.attn_qkv.weight", "offset": 7467417600, "shape": [ 5120, 7680 ], "size": 18022400, "type": 12 }, { "name": "output_norm.weight", "offset": 7489536000, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }