phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 3, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 131072, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238208, "tensors": [ { "name": "rope_factors_short.weight", "offset": 256, "shape": [ 64 ], "size": 256, "type": 0 }, { "name": "token_embd.weight", "offset": 512, "shape": [ 5120, 32064 ], "size": 256, "type": 3 }, { "name": "blk.0.attn_norm.weight", "offset": 102605312, "shape": [ 5120 ], "size": 102604800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 102625792, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.0.ffn_up.weight", "offset": 159969792, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.0.ffn_norm.weight", "offset": 274657792, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 274678272, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.0.attn_qkv.weight", "offset": 291062272, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.1.attn_norm.weight", "offset": 315638272, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 315658752, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.1.ffn_up.weight", "offset": 373002752, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.1.ffn_norm.weight", "offset": 487690752, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 487711232, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.1.attn_qkv.weight", "offset": 504095232, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.2.attn_norm.weight", "offset": 528671232, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 528691712, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.2.ffn_up.weight", "offset": 586035712, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.2.ffn_norm.weight", "offset": 700723712, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 700744192, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.2.attn_qkv.weight", "offset": 717128192, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.3.attn_norm.weight", "offset": 741704192, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 741724672, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.3.ffn_up.weight", "offset": 799068672, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.3.ffn_norm.weight", "offset": 913756672, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 913777152, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.3.attn_qkv.weight", "offset": 930161152, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.4.attn_norm.weight", "offset": 954737152, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 954757632, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.4.ffn_up.weight", "offset": 1012101632, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.4.ffn_norm.weight", "offset": 1126789632, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1126810112, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.4.attn_qkv.weight", "offset": 1143194112, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.5.attn_norm.weight", "offset": 1167770112, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1167790592, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.5.ffn_up.weight", "offset": 1225134592, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.5.ffn_norm.weight", "offset": 1339822592, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1339843072, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.5.attn_qkv.weight", "offset": 1356227072, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.6.ffn_up.weight", "offset": 1380803072, "shape": [ 5120, 35840 ], "size": 24576000, "type": 3 }, { "name": "blk.6.attn_output.weight", "offset": 1495491072, "shape": [ 5120, 5120 ], "size": 114688000, "type": 3 }, { "name": "blk.6.attn_qkv.weight", "offset": 1511875072, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.10.attn_norm.weight", "offset": 1536451072, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1536471552, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.10.ffn_up.weight", "offset": 1593815552, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.10.ffn_norm.weight", "offset": 1708503552, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1708524032, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.10.attn_qkv.weight", "offset": 1724908032, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.11.attn_norm.weight", "offset": 1749484032, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1749504512, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.11.ffn_up.weight", "offset": 1806848512, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.11.ffn_norm.weight", "offset": 1921536512, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1921556992, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.11.attn_qkv.weight", "offset": 1937940992, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.12.attn_norm.weight", "offset": 1962516992, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1962537472, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.12.ffn_up.weight", "offset": 2019881472, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.12.ffn_norm.weight", "offset": 2134569472, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2134589952, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.12.attn_qkv.weight", "offset": 2150973952, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.13.attn_norm.weight", "offset": 2175549952, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2175570432, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.13.ffn_up.weight", "offset": 2232914432, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.13.ffn_norm.weight", "offset": 2347602432, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2347622912, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.13.attn_qkv.weight", "offset": 2364006912, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.6.attn_norm.weight", "offset": 2388582912, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2388603392, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.6.ffn_norm.weight", "offset": 2445947392, "shape": [ 5120 ], "size": 57344000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2445967872, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2445988352, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.7.ffn_up.weight", "offset": 2503332352, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.7.ffn_norm.weight", "offset": 2618020352, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2618040832, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.7.attn_qkv.weight", "offset": 2634424832, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.8.attn_norm.weight", "offset": 2659000832, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2659021312, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.8.ffn_up.weight", "offset": 2716365312, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.8.ffn_norm.weight", "offset": 2831053312, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2831073792, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.8.attn_qkv.weight", "offset": 2847457792, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.9.attn_norm.weight", "offset": 2872033792, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2872054272, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.9.ffn_up.weight", "offset": 2929398272, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.9.ffn_norm.weight", "offset": 3044086272, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3044106752, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.9.attn_qkv.weight", "offset": 3060490752, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.14.attn_norm.weight", "offset": 3085066752, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 3085087232, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.14.ffn_up.weight", "offset": 3142431232, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.14.ffn_norm.weight", "offset": 3257119232, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 3257139712, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.14.attn_qkv.weight", "offset": 3273523712, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.15.attn_norm.weight", "offset": 3298099712, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 3298120192, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.15.ffn_up.weight", "offset": 3355464192, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.15.ffn_norm.weight", "offset": 3470152192, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3470172672, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.15.attn_qkv.weight", "offset": 3486556672, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.16.attn_norm.weight", "offset": 3511132672, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3511153152, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.16.ffn_up.weight", "offset": 3568497152, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.16.ffn_norm.weight", "offset": 3683185152, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3683205632, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.16.attn_qkv.weight", "offset": 3699589632, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.17.attn_norm.weight", "offset": 3724165632, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3724186112, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.17.ffn_up.weight", "offset": 3781530112, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.17.ffn_norm.weight", "offset": 3896218112, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3896238592, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.17.attn_qkv.weight", "offset": 3912622592, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.18.attn_norm.weight", "offset": 3937198592, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3937219072, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.18.ffn_up.weight", "offset": 3994563072, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.18.ffn_norm.weight", "offset": 4109251072, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 4109271552, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.18.attn_qkv.weight", "offset": 4125655552, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.19.attn_norm.weight", "offset": 4150231552, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 4150252032, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.19.ffn_up.weight", "offset": 4207596032, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.19.ffn_norm.weight", "offset": 4322284032, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 4322304512, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.19.attn_qkv.weight", "offset": 4338688512, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.20.attn_norm.weight", "offset": 4363264512, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 4363284992, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.20.ffn_up.weight", "offset": 4420628992, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.20.ffn_norm.weight", "offset": 4535316992, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4535337472, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.20.attn_qkv.weight", "offset": 4551721472, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.21.attn_output.weight", "offset": 4576297472, "shape": [ 5120, 5120 ], "size": 24576000, "type": 3 }, { "name": "blk.21.attn_qkv.weight", "offset": 4592681472, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.21.attn_norm.weight", "offset": 4617257472, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4617277952, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.21.ffn_up.weight", "offset": 4674621952, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.21.ffn_norm.weight", "offset": 4789309952, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4789330432, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4789350912, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.22.ffn_up.weight", "offset": 4846694912, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.22.ffn_norm.weight", "offset": 4961382912, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4961403392, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.22.attn_qkv.weight", "offset": 4977787392, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.23.attn_norm.weight", "offset": 5002363392, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5002383872, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.23.ffn_up.weight", "offset": 5059727872, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.23.ffn_norm.weight", "offset": 5174415872, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5174436352, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.23.attn_qkv.weight", "offset": 5190820352, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.24.attn_norm.weight", "offset": 5215396352, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5215416832, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.24.ffn_up.weight", "offset": 5272760832, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.24.ffn_norm.weight", "offset": 5387448832, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5387469312, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.24.attn_qkv.weight", "offset": 5403853312, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.25.attn_norm.weight", "offset": 5428429312, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 5428449792, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.25.ffn_up.weight", "offset": 5485793792, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.25.ffn_norm.weight", "offset": 5600481792, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5600502272, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.25.attn_qkv.weight", "offset": 5616886272, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.26.attn_norm.weight", "offset": 5641462272, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5641482752, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.26.ffn_up.weight", "offset": 5698826752, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.26.ffn_norm.weight", "offset": 5813514752, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5813535232, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.26.attn_qkv.weight", "offset": 5829919232, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.27.attn_norm.weight", "offset": 5854495232, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5854515712, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.27.ffn_up.weight", "offset": 5911859712, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.27.ffn_norm.weight", "offset": 6026547712, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6026568192, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.27.attn_qkv.weight", "offset": 6042952192, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.28.attn_output.weight", "offset": 6067528192, "shape": [ 5120, 5120 ], "size": 24576000, "type": 3 }, { "name": "blk.28.attn_qkv.weight", "offset": 6083912192, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.28.attn_norm.weight", "offset": 6108488192, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6108508672, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.28.ffn_up.weight", "offset": 6165852672, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.28.ffn_norm.weight", "offset": 6280540672, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 6280561152, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6280581632, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.29.ffn_up.weight", "offset": 6337925632, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.29.ffn_norm.weight", "offset": 6452613632, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 6452634112, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.29.attn_qkv.weight", "offset": 6469018112, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.30.attn_norm.weight", "offset": 6493594112, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 6493614592, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.30.ffn_up.weight", "offset": 6550958592, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.30.ffn_norm.weight", "offset": 6665646592, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 6665667072, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "|x\ufffd\ufffd(K\u0011!\ufffd\ufffdFU\ufffd\ufffd\ufffdd\ufffd\t\ufffdF\ufffdT", "offset": 6682051072, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.31.attn_norm.weight", "offset": 6706627072, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6706647552, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.31.ffn_up.weight", "offset": 6763991552, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.31.ffn_norm.weight", "offset": 6878679552, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6878700032, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.31.attn_qkv.weight", "offset": 6895084032, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.32.attn_norm.weight", "offset": 6919660032, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6919680512, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.32.ffn_up.weight", "offset": 6977024512, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.32.ffn_norm.weight", "offset": 7091712512, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 7091732992, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.32.attn_qkv.weight", "offset": 7108116992, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.33.attn_norm.weight", "offset": 7132692992, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 7132713472, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.33.ffn_up.weight", "offset": 7190057472, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.33.ffn_norm.weight", "offset": 7304745472, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 7304765952, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.33.attn_qkv.weight", "offset": 7321149952, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.34.attn_norm.weight", "offset": 7345725952, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 7345746432, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.34.ffn_up.weight", "offset": 7403090432, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.34.ffn_norm.weight", "offset": 7517778432, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 7517798912, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.34.attn_qkv.weight", "offset": 7534182912, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.35.attn_output.weight", "offset": 7558758912, "shape": [ 5120, 5120 ], "size": 24576000, "type": 3 }, { "name": "blk.35.attn_qkv.weight", "offset": 7575142912, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "output.weight", "offset": 7599718912, "shape": [ 5120, 32064 ], "size": 24576000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 7734387712, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 7734408192, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.35.ffn_up.weight", "offset": 7791752192, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.35.ffn_norm.weight", "offset": 7906440192, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7906460672, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7906481152, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.36.ffn_up.weight", "offset": 7963825152, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.36.ffn_norm.weight", "offset": 8078513152, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 8078533632, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.36.attn_qkv.weight", "offset": 8094917632, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.37.attn_norm.weight", "offset": 8119493632, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 8119514112, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.37.ffn_up.weight", "offset": 8176858112, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.37.ffn_norm.weight", "offset": 8291546112, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 8291566592, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.37.attn_qkv.weight", "offset": 8307950592, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.38.attn_norm.weight", "offset": 8332526592, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 8332547072, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.38.ffn_up.weight", "offset": 8389891072, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.38.ffn_norm.weight", "offset": 8504579072, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 8504599552, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.38.attn_qkv.weight", "offset": 8520983552, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "blk.39.attn_norm.weight", "offset": 8545559552, "shape": [ 5120 ], "size": 24576000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 8545580032, "shape": [ 17920, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.39.ffn_up.weight", "offset": 8602924032, "shape": [ 5120, 35840 ], "size": 57344000, "type": 3 }, { "name": "blk.39.ffn_norm.weight", "offset": 8717612032, "shape": [ 5120 ], "size": 114688000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 8717632512, "shape": [ 5120, 5120 ], "size": 20480, "type": 3 }, { "name": "blk.39.attn_qkv.weight", "offset": 8734016512, "shape": [ 5120, 7680 ], "size": 16384000, "type": 3 }, { "name": "output_norm.weight", "offset": 8758592512, "shape": [ 5120 ], "size": 24576000, "type": 0 } ], "version": 3 }