phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 2, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 32, "phi3.attention.head_count_kv": 32, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 32, "phi3.context_length": 131072, "phi3.embedding_length": 3072, "phi3.feed_forward_length": 8192, "phi3.rope.dimension_count": 96, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 3821079648, "tensors": [ { "name": "rope_factors_short.weight", "offset": 192, "shape": [ 48 ], "size": 192, "type": 0 }, { "name": "token_embd.weight", "offset": 384, "shape": [ 3072, 32064 ], "size": 192, "type": 2 }, { "name": "blk.0.attn_norm.weight", "offset": 55406976, "shape": [ 3072 ], "size": 55406592, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 55419264, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 69575040, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 97886592, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 97898880, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.0.attn_qkv.weight", "offset": 103207296, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 119132544, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 119144832, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 133300608, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 161612160, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 161624448, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.1.attn_qkv.weight", "offset": 166932864, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 182858112, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 182870400, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 197026176, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 225337728, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 225350016, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.10.attn_qkv.weight", "offset": 230658432, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 246583680, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 246595968, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 260751744, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 289063296, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 289075584, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.11.attn_qkv.weight", "offset": 294384000, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 310309248, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 310321536, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 324477312, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 352788864, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 352801152, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.12.attn_qkv.weight", "offset": 358109568, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 374034816, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 374047104, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 388202880, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 416514432, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 416526720, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.13.attn_qkv.weight", "offset": 421835136, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 437760384, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 437772672, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 451928448, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 480240000, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 480252288, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.14.attn_qkv.weight", "offset": 485560704, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 501485952, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 501498240, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 515654016, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 543965568, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 543977856, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.15.attn_qkv.weight", "offset": 549286272, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 565211520, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 565223808, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 579379584, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 607691136, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 607703424, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.16.attn_qkv.weight", "offset": 613011840, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 628937088, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 628949376, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 643105152, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 671416704, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 671428992, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.17.attn_qkv.weight", "offset": 676737408, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 692662656, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 692674944, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 706830720, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 735142272, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 735154560, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.18.attn_qkv.weight", "offset": 740462976, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 756388224, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 756400512, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 770556288, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 798867840, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 798880128, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.19.attn_qkv.weight", "offset": 804188544, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 820113792, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 820126080, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 834281856, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 862593408, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 862605696, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.2.attn_qkv.weight", "offset": 867914112, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 883839360, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 883851648, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 898007424, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 926318976, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 926331264, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.20.attn_qkv.weight", "offset": 931639680, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 947564928, "shape": [ 3072, 3072 ], "size": 15925248, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 952873344, "shape": [ 3072 ], "size": 5308416, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 952885632, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 967041408, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 995352960, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 995365248, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.3.attn_qkv.weight", "offset": 1000673664, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 1016598912, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1016611200, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 1030766976, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 1059078528, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1059090816, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.4.attn_qkv.weight", "offset": 1064399232, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 1080324480, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1080336768, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 1094492544, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1122804096, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1122816384, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.5.attn_qkv.weight", "offset": 1128124800, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 1144050048, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 1144062336, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 1158218112, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 1186529664, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.6.attn_output.weight", "offset": 1186541952, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.6.attn_qkv.weight", "offset": 1191850368, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.7.attn_norm.weight", "offset": 1207775616, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 1207787904, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 1221943680, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 1250255232, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 1250267520, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.7.attn_qkv.weight", "offset": 1255575936, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 1271501184, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 1271513472, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 1285669248, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 1313980800, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 1313993088, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.8.attn_qkv.weight", "offset": 1319301504, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 1335226752, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 1335239040, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 1349394816, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 1377706368, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 1377718656, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.9.attn_qkv.weight", "offset": 1383027072, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "output.weight", "offset": 1398952320, "shape": [ 3072, 32064 ], "size": 15925248, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 1479753600, "shape": [ 3072 ], "size": 80801280, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 1479765888, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 1493921664, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 1522233216, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.21.attn_qkv.weight", "offset": 1522245504, "shape": [ 3072, 9216 ], "size": 12288, "type": 2 }, { "name": "blk.22.attn_norm.weight", "offset": 1538170752, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 1538183040, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 1552338816, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 1580650368, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 1580662656, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.22.attn_qkv.weight", "offset": 1585971072, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 1601896320, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 1601908608, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 1616064384, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 1644375936, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 1644388224, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.23.attn_qkv.weight", "offset": 1649696640, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 1665621888, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 1665634176, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 1679789952, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 1708101504, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 1708113792, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.24.attn_qkv.weight", "offset": 1713422208, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 1729347456, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 1729359744, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 1743515520, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 1771827072, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 1771839360, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.25.attn_qkv.weight", "offset": 1777147776, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 1793073024, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 1793085312, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 1807241088, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 1835552640, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 1835564928, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.26.attn_qkv.weight", "offset": 1840873344, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 1856798592, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 1856810880, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 1870966656, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 1899278208, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 1899290496, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.27.attn_qkv.weight", "offset": 1904598912, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 1920524160, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 1920536448, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 1934692224, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 1963003776, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.28.attn_output.weight", "offset": 1963016064, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.28.attn_qkv.weight", "offset": 1968324480, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.29.attn_norm.weight", "offset": 1984249728, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 1984262016, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 1998417792, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 2026729344, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 2026741632, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.29.attn_qkv.weight", "offset": 2032050048, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 2047975296, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 2047987584, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 2062143360, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 2090454912, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 2090467200, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.30.attn_qkv.weight", "offset": 2095775616, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 2111700864, "shape": [ 3072 ], "size": 15925248, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 2111713152, "shape": [ 8192, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 2125868928, "shape": [ 3072, 16384 ], "size": 14155776, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 2154180480, "shape": [ 3072 ], "size": 28311552, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 2154192768, "shape": [ 3072, 3072 ], "size": 12288, "type": 2 }, { "name": "blk.31.attn_qkv.weight", "offset": 2159501184, "shape": [ 3072, 9216 ], "size": 5308416, "type": 2 }, { "name": "output_norm.weight", "offset": 2175426432, "shape": [ 3072 ], "size": 15925248, "type": 0 } ], "version": 3 }