phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 12, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 70540800, "shape": [ 5120 ], "size": 70540800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 70561280, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 133639680, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 212487680, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 212508160, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.0.attn_qkv.weight", "offset": 227253760, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 249372160, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 249392640, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 312471040, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 391319040, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 391339520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.1.attn_qkv.weight", "offset": 406085120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 428203520, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 428224000, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 479833600, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 558681600, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 558702080, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.attn_qkv.weight", "offset": 573447680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 595566080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 595586560, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 647196160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 726044160, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 726064640, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.attn_qkv.weight", "offset": 740810240, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 762928640, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 762949120, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 814558720, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 893406720, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 893427200, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.attn_qkv.weight", "offset": 908172800, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 930291200, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 930311680, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 981921280, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1060769280, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1060789760, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.attn_qkv.weight", "offset": 1075535360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1097653760, "shape": [ 5120, 35840 ], "size": 22118400, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1176501760, "shape": [ 5120, 5120 ], "size": 78848000, "type": 12 }, { "name": "blk.6.attn_qkv.weight", "offset": 1191247360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1213365760, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1213386240, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1264995840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1343843840, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1343864320, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.attn_qkv.weight", "offset": 1358609920, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1380728320, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1380748800, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1432358400, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1511206400, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1511226880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.attn_qkv.weight", "offset": 1525972480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1548090880, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1548111360, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1599720960, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1678568960, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1678589440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.attn_qkv.weight", "offset": 1693335040, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1715453440, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1715473920, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1767083520, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1845931520, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1845952000, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.attn_qkv.weight", "offset": 1860697600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 1882816000, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1882836480, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 1934446080, "shape": [ 5120 ], "size": 51609600, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1934466560, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1934487040, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 1986096640, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 2064944640, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2064965120, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.attn_qkv.weight", "offset": 2079710720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2101829120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2101849600, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2153459200, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 2232307200, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2232327680, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.attn_qkv.weight", "offset": 2247073280, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2269191680, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2269212160, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2320821760, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2399669760, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2399690240, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.attn_qkv.weight", "offset": 2414435840, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2436554240, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2436574720, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 2488184320, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2567032320, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2567052800, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.attn_qkv.weight", "offset": 2581798400, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 2603916800, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2603937280, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 2655546880, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2734394880, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2734415360, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.attn_qkv.weight", "offset": 2749160960, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2771279360, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2771299840, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 2822909440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2901757440, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2901777920, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.attn_qkv.weight", "offset": 2916523520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2938641920, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2938662400, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 2990272000, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 3069120000, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3069140480, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.attn_qkv.weight", "offset": 3083886080, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3106004480, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3106024960, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3157634560, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 3236482560, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3236503040, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.attn_qkv.weight", "offset": 3251248640, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 3273367040, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3273387520, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 3324997120, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.19.ffn_norm.weight", "offset": 3403845120, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3403865600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.attn_qkv.weight", "offset": 3418611200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3440729600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3440750080, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 3492359680, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 3571207680, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3571228160, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.attn_qkv.weight", "offset": 3585973760, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 3608092160, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.21.attn_qkv.weight", "offset": 3622837760, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 3644956160, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3644976640, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 3696586240, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 3775434240, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 3775454720, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3775475200, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 3827084800, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 3905932800, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 3905953280, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.attn_qkv.weight", "offset": 3920698880, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 3942817280, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3942837760, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 3994447360, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 4073295360, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4073315840, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.attn_qkv.weight", "offset": 4088061440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4110179840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4110200320, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 4161809920, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 4240657920, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4240678400, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.attn_qkv.weight", "offset": 4255424000, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4277542400, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4277562880, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 4329172480, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 4408020480, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 4408040960, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.attn_qkv.weight", "offset": 4422786560, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 4444904960, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4444925440, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 4496535040, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 4575383040, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 4575403520, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.attn_qkv.weight", "offset": 4590149120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 4612267520, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4612288000, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 4663897600, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 4742745600, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 4742766080, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.attn_qkv.weight", "offset": 4757511680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 4779630080, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.28.attn_qkv.weight", "offset": 4794375680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 4816494080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4816514560, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 4868124160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 4946972160, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 4946992640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4947013120, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 4998622720, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 5077470720, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5077491200, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.attn_qkv.weight", "offset": 5092236800, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 5114355200, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5114375680, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 5165985280, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 5244833280, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 5244853760, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.attn_qkv.weight", "offset": 5259599360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 5281717760, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5281738240, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 5333347840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 5412195840, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 5412216320, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.attn_qkv.weight", "offset": 5426961920, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 5449080320, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 5449100800, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 5500710400, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 5579558400, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 5579578880, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.attn_qkv.weight", "offset": 5594324480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 5616442880, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 5616463360, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 5668072960, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 5746920960, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 5746941440, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.attn_qkv.weight", "offset": 5761687040, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 5783805440, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 5783825920, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 5835435520, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 5914283520, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 5914304000, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.attn_qkv.weight", "offset": 5929049600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 5951168000, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.35.attn_qkv.weight", "offset": 5965913600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output.weight", "offset": 5988032000, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 6122700800, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6122721280, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 6174330880, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 6253178880, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 6253199360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6253219840, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 6304829440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 6383677440, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 6383697920, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.attn_qkv.weight", "offset": 6398443520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 6420561920, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 6420582400, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 6472192000, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 6551040000, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 6551060480, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.attn_qkv.weight", "offset": 6565806080, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 6587924480, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 6587944960, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 6639554560, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 6718402560, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 6718423040, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.attn_qkv.weight", "offset": 6733168640, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 6755287040, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 6755307520, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 6806917120, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 6885765120, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 6885785600, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.attn_qkv.weight", "offset": 6900531200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output_norm.weight", "offset": 6922649600, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }