phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 12, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 11 }, { "name": "blk.0.attn_norm.weight", "offset": 70541312, "shape": [ 5120 ], "size": 70540800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 70561792, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.0.ffn_up.weight", "offset": 133640192, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.0.ffn_norm.weight", "offset": 212488192, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 212508672, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.0.attn_qkv.weight", "offset": 227254272, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.1.attn_norm.weight", "offset": 249372672, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 249393152, "shape": [ 17920, 5120 ], "size": 20480, "type": 13 }, { "name": "blk.1.ffn_up.weight", "offset": 312471552, "shape": [ 5120, 35840 ], "size": 63078400, "type": 11 }, { "name": "blk.1.ffn_norm.weight", "offset": 391319552, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 391340032, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.1.attn_qkv.weight", "offset": 406085632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.2.attn_norm.weight", "offset": 428204032, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 428224512, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.ffn_up.weight", "offset": 479834112, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.2.ffn_norm.weight", "offset": 558682112, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 558702592, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.2.attn_qkv.weight", "offset": 573448192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.3.attn_norm.weight", "offset": 595566592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 595587072, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.ffn_up.weight", "offset": 647196672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.3.ffn_norm.weight", "offset": 726044672, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 726065152, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.3.attn_qkv.weight", "offset": 740810752, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.4.attn_norm.weight", "offset": 762929152, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 762949632, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.ffn_up.weight", "offset": 814559232, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.4.ffn_norm.weight", "offset": 893407232, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 893427712, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.4.attn_qkv.weight", "offset": 908173312, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.5.attn_norm.weight", "offset": 930291712, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 930312192, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.ffn_up.weight", "offset": 981921792, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.5.ffn_norm.weight", "offset": 1060769792, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1060790272, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.5.attn_qkv.weight", "offset": 1075535872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.ffn_up.weight", "offset": 1097654272, "shape": [ 5120, 35840 ], "size": 22118400, "type": 11 }, { "name": "blk.6.attn_output.weight", "offset": 1176502272, "shape": [ 5120, 5120 ], "size": 78848000, "type": 12 }, { "name": "blk.6.attn_qkv.weight", "offset": 1191247872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.10.attn_norm.weight", "offset": 1213366272, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1213386752, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.ffn_up.weight", "offset": 1264996352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.10.ffn_norm.weight", "offset": 1343844352, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1343864832, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.10.attn_qkv.weight", "offset": 1358610432, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.11.attn_norm.weight", "offset": 1380728832, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1380749312, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.ffn_up.weight", "offset": 1432358912, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.11.ffn_norm.weight", "offset": 1511206912, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1511227392, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.11.attn_qkv.weight", "offset": 1525972992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.12.attn_norm.weight", "offset": 1548091392, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1548111872, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.ffn_up.weight", "offset": 1599721472, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.12.ffn_norm.weight", "offset": 1678569472, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1678589952, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.12.attn_qkv.weight", "offset": 1693335552, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.13.attn_norm.weight", "offset": 1715453952, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1715474432, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.ffn_up.weight", "offset": 1767084032, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.13.ffn_norm.weight", "offset": 1845932032, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1845952512, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.13.attn_qkv.weight", "offset": 1860698112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.6.attn_norm.weight", "offset": 1882816512, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1882836992, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.6.ffn_norm.weight", "offset": 1934446592, "shape": [ 5120 ], "size": 51609600, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 1934467072, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1934487552, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.ffn_up.weight", "offset": 1986097152, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.7.ffn_norm.weight", "offset": 2064945152, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2064965632, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.7.attn_qkv.weight", "offset": 2079711232, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.8.attn_norm.weight", "offset": 2101829632, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2101850112, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.ffn_up.weight", "offset": 2153459712, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.8.ffn_norm.weight", "offset": 2232307712, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2232328192, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.8.attn_qkv.weight", "offset": 2247073792, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.9.attn_norm.weight", "offset": 2269192192, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2269212672, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.ffn_up.weight", "offset": 2320822272, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.9.ffn_norm.weight", "offset": 2399670272, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2399690752, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.9.attn_qkv.weight", "offset": 2414436352, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.14.attn_norm.weight", "offset": 2436554752, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2436575232, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.ffn_up.weight", "offset": 2488184832, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.14.ffn_norm.weight", "offset": 2567032832, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2567053312, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.14.attn_qkv.weight", "offset": 2581798912, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.15.attn_norm.weight", "offset": 2603917312, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2603937792, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.ffn_up.weight", "offset": 2655547392, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.15.ffn_norm.weight", "offset": 2734395392, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 2734415872, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.15.attn_qkv.weight", "offset": 2749161472, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.16.attn_norm.weight", "offset": 2771279872, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2771300352, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.ffn_up.weight", "offset": 2822909952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.16.ffn_norm.weight", "offset": 2901757952, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2901778432, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.16.attn_qkv.weight", "offset": 2916524032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.17.attn_norm.weight", "offset": 2938642432, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2938662912, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.ffn_up.weight", "offset": 2990272512, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.17.ffn_norm.weight", "offset": 3069120512, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3069140992, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.17.attn_qkv.weight", "offset": 3083886592, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.18.attn_norm.weight", "offset": 3106004992, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3106025472, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.ffn_up.weight", "offset": 3157635072, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.18.ffn_norm.weight", "offset": 3236483072, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3236503552, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.18.attn_qkv.weight", "offset": 3251249152, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.19.attn_norm.weight", "offset": 3273367552, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3273388032, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.ffn_up.weight", "offset": 3324997632, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "ht\u0002\u0000\u0000\u0000\u0000F\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000", "offset": 3403845632, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3403866112, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.19.attn_qkv.weight", "offset": 3418611712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.20.attn_norm.weight", "offset": 3440730112, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3440750592, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.ffn_up.weight", "offset": 3492360192, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.20.ffn_norm.weight", "offset": 3571208192, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3571228672, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.20.attn_qkv.weight", "offset": 3585974272, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_output.weight", "offset": 3608092672, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.21.attn_qkv.weight", "offset": 3622838272, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.21.attn_norm.weight", "offset": 3644956672, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 3644977152, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.21.ffn_up.weight", "offset": 3696586752, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.21.ffn_norm.weight", "offset": 3775434752, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 3775455232, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 3775475712, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.ffn_up.weight", "offset": 3827085312, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.22.ffn_norm.weight", "offset": 3905933312, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 3905953792, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.22.attn_qkv.weight", "offset": 3920699392, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.23.attn_norm.weight", "offset": 3942817792, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 3942838272, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.ffn_up.weight", "offset": 3994447872, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.23.ffn_norm.weight", "offset": 4073295872, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4073316352, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.23.attn_qkv.weight", "offset": 4088061952, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.24.attn_norm.weight", "offset": 4110180352, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4110200832, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.ffn_up.weight", "offset": 4161810432, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.24.ffn_norm.weight", "offset": 4240658432, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4240678912, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.24.attn_qkv.weight", "offset": 4255424512, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.25.attn_norm.weight", "offset": 4277542912, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4277563392, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.ffn_up.weight", "offset": 4329172992, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.25.ffn_norm.weight", "offset": 4408020992, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 4408041472, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.25.attn_qkv.weight", "offset": 4422787072, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.26.attn_norm.weight", "offset": 4444905472, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 4444925952, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.ffn_up.weight", "offset": 4496535552, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.26.ffn_norm.weight", "offset": 4575383552, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 4575404032, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.26.attn_qkv.weight", "offset": 4590149632, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.27.attn_norm.weight", "offset": 4612268032, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 4612288512, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.ffn_up.weight", "offset": 4663898112, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.27.ffn_norm.weight", "offset": 4742746112, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 4742766592, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.27.attn_qkv.weight", "offset": 4757512192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_output.weight", "offset": 4779630592, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.28.attn_qkv.weight", "offset": 4794376192, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.28.attn_norm.weight", "offset": 4816494592, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 4816515072, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.28.ffn_up.weight", "offset": 4868124672, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.28.ffn_norm.weight", "offset": 4946972672, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 4946993152, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 4947013632, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.ffn_up.weight", "offset": 4998623232, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.29.ffn_norm.weight", "offset": 5077471232, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5077491712, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.29.attn_qkv.weight", "offset": 5092237312, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.30.attn_norm.weight", "offset": 5114355712, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5114376192, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.ffn_up.weight", "offset": 5165985792, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.30.ffn_norm.weight", "offset": 5244833792, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 5244854272, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.30.attn_qkv.weight", "offset": 5259599872, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.31.attn_norm.weight", "offset": 5281718272, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 5281738752, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.ffn_up.weight", "offset": 5333348352, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.31.ffn_norm.weight", "offset": 5412196352, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 5412216832, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.31.attn_qkv.weight", "offset": 5426962432, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.32.attn_norm.weight", "offset": 5449080832, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 5449101312, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.ffn_up.weight", "offset": 5500710912, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.32.ffn_norm.weight", "offset": 5579558912, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 5579579392, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.32.attn_qkv.weight", "offset": 5594324992, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.33.attn_norm.weight", "offset": 5616443392, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 5616463872, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.ffn_up.weight", "offset": 5668073472, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.33.ffn_norm.weight", "offset": 5746921472, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 5746941952, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.33.attn_qkv.weight", "offset": 5761687552, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.34.attn_norm.weight", "offset": 5783805952, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 5783826432, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.ffn_up.weight", "offset": 5835436032, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.34.ffn_norm.weight", "offset": 5914284032, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 5914304512, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.34.attn_qkv.weight", "offset": 5929050112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.35.attn_output.weight", "offset": 5951168512, "shape": [ 5120, 5120 ], "size": 22118400, "type": 12 }, { "name": "blk.35.attn_qkv.weight", "offset": 5965914112, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output.weight", "offset": 5988032512, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 6122701312, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6122721792, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.35.ffn_up.weight", "offset": 6174331392, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.35.ffn_norm.weight", "offset": 6253179392, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 6253199872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 6253220352, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.ffn_up.weight", "offset": 6304829952, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.36.ffn_norm.weight", "offset": 6383677952, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 6383698432, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.36.attn_qkv.weight", "offset": 6398444032, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.37.attn_norm.weight", "offset": 6420562432, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 6420582912, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.ffn_up.weight", "offset": 6472192512, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.37.ffn_norm.weight", "offset": 6551040512, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 6551060992, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.37.attn_qkv.weight", "offset": 6565806592, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.38.attn_norm.weight", "offset": 6587924992, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 6587945472, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.ffn_up.weight", "offset": 6639555072, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.38.ffn_norm.weight", "offset": 6718403072, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 6718423552, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.38.attn_qkv.weight", "offset": 6733169152, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "blk.39.attn_norm.weight", "offset": 6755287552, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 6755308032, "shape": [ 17920, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.ffn_up.weight", "offset": 6806917632, "shape": [ 5120, 35840 ], "size": 51609600, "type": 11 }, { "name": "blk.39.ffn_norm.weight", "offset": 6885765632, "shape": [ 5120 ], "size": 78848000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 6885786112, "shape": [ 5120, 5120 ], "size": 20480, "type": 12 }, { "name": "blk.39.attn_qkv.weight", "offset": 6900531712, "shape": [ 5120, 7680 ], "size": 14745600, "type": 12 }, { "name": "output_norm.weight", "offset": 6922650112, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }