phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 1, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 32, "phi3.attention.head_count_kv": 32, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 32, "phi3.context_length": 131072, "phi3.embedding_length": 3072, "phi3.feed_forward_length": 8192, "phi3.rope.dimension_count": 96, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.attn_factor": 1.1902381, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": true, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 3821079648, "tensors": [ { "name": "rope_factors_short.weight", "offset": 192, "shape": [ 48 ], "size": 192, "type": 0 }, { "name": "token_embd.weight", "offset": 384, "shape": [ 3072, 32064 ], "size": 192, "type": 1 }, { "name": "blk.0.attn_norm.weight", "offset": 197001600, "shape": [ 3072 ], "size": 197001216, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 197013888, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.0.ffn_up.weight", "offset": 247345536, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.0.ffn_norm.weight", "offset": 348008832, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 348021120, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.0.attn_qkv.weight", "offset": 366895488, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.1.attn_norm.weight", "offset": 423518592, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 423530880, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.1.ffn_up.weight", "offset": 473862528, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.1.ffn_norm.weight", "offset": 574525824, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 574538112, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.1.attn_qkv.weight", "offset": 593412480, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.10.attn_norm.weight", "offset": 650035584, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 650047872, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.10.ffn_up.weight", "offset": 700379520, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.10.ffn_norm.weight", "offset": 801042816, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 801055104, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.10.attn_qkv.weight", "offset": 819929472, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.11.attn_norm.weight", "offset": 876552576, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 876564864, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.11.ffn_up.weight", "offset": 926896512, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.11.ffn_norm.weight", "offset": 1027559808, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1027572096, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.11.attn_qkv.weight", "offset": 1046446464, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.12.attn_norm.weight", "offset": 1103069568, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1103081856, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.12.ffn_up.weight", "offset": 1153413504, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.12.ffn_norm.weight", "offset": 1254076800, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1254089088, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.12.attn_qkv.weight", "offset": 1272963456, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.13.attn_norm.weight", "offset": 1329586560, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1329598848, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.13.ffn_up.weight", "offset": 1379930496, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.13.ffn_norm.weight", "offset": 1480593792, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 1480606080, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.13.attn_qkv.weight", "offset": 1499480448, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.14.attn_norm.weight", "offset": 1556103552, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 1556115840, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.14.ffn_up.weight", "offset": 1606447488, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.14.ffn_norm.weight", "offset": 1707110784, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 1707123072, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.14.attn_qkv.weight", "offset": 1725997440, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.15.attn_norm.weight", "offset": 1782620544, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 1782632832, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.15.ffn_up.weight", "offset": 1832964480, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.15.ffn_norm.weight", "offset": 1933627776, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 1933640064, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.15.attn_qkv.weight", "offset": 1952514432, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.16.attn_norm.weight", "offset": 2009137536, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 2009149824, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.16.ffn_up.weight", "offset": 2059481472, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.16.ffn_norm.weight", "offset": 2160144768, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 2160157056, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.16.attn_qkv.weight", "offset": 2179031424, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.17.attn_norm.weight", "offset": 2235654528, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 2235666816, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.17.ffn_up.weight", "offset": 2285998464, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.17.ffn_norm.weight", "offset": 2386661760, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 2386674048, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.17.attn_qkv.weight", "offset": 2405548416, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.18.attn_norm.weight", "offset": 2462171520, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 2462183808, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.18.ffn_up.weight", "offset": 2512515456, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.18.ffn_norm.weight", "offset": 2613178752, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 2613191040, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.18.attn_qkv.weight", "offset": 2632065408, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.19.attn_norm.weight", "offset": 2688688512, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 2688700800, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.19.ffn_up.weight", "offset": 2739032448, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.19.ffn_norm.weight", "offset": 2839695744, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 2839708032, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.19.attn_qkv.weight", "offset": 2858582400, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.2.attn_norm.weight", "offset": 2915205504, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 2915217792, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.2.ffn_up.weight", "offset": 2965549440, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.2.ffn_norm.weight", "offset": 3066212736, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 3066225024, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.2.attn_qkv.weight", "offset": 3085099392, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.20.attn_norm.weight", "offset": 3141722496, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3141734784, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.20.ffn_up.weight", "offset": 3192066432, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.20.ffn_norm.weight", "offset": 3292729728, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 3292742016, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.20.attn_qkv.weight", "offset": 3311616384, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.21.attn_output.weight", "offset": 3368239488, "shape": [ 3072, 3072 ], "size": 56623104, "type": 1 }, { "name": "blk.3.attn_norm.weight", "offset": 3387113856, "shape": [ 3072 ], "size": 18874368, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 3387126144, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.3.ffn_up.weight", "offset": 3437457792, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.3.ffn_norm.weight", "offset": 3538121088, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 3538133376, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.3.attn_qkv.weight", "offset": 3557007744, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.4.attn_norm.weight", "offset": 3613630848, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 3613643136, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.4.ffn_up.weight", "offset": 3663974784, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.4.ffn_norm.weight", "offset": 3764638080, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 3764650368, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.4.attn_qkv.weight", "offset": 3783524736, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.5.attn_norm.weight", "offset": 3840147840, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 3840160128, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.5.ffn_up.weight", "offset": 3890491776, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.5.ffn_norm.weight", "offset": 3991155072, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 3991167360, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.5.attn_qkv.weight", "offset": 4010041728, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.6.attn_norm.weight", "offset": 4066664832, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4066677120, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.6.ffn_up.weight", "offset": 4117008768, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.6.ffn_norm.weight", "offset": 4217672064, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.6.attn_output.weight", "offset": 4217684352, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.6.attn_qkv.weight", "offset": 4236558720, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.7.attn_norm.weight", "offset": 4293181824, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4293194112, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.7.ffn_up.weight", "offset": 4343525760, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.7.ffn_norm.weight", "offset": 4444189056, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 4444201344, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.7.attn_qkv.weight", "offset": 4463075712, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.8.attn_norm.weight", "offset": 4519698816, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4519711104, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.8.ffn_up.weight", "offset": 4570042752, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.8.ffn_norm.weight", "offset": 4670706048, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 4670718336, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.8.attn_qkv.weight", "offset": 4689592704, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.9.attn_norm.weight", "offset": 4746215808, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 4746228096, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.9.ffn_up.weight", "offset": 4796559744, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.9.ffn_norm.weight", "offset": 4897223040, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 4897235328, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.9.attn_qkv.weight", "offset": 4916109696, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "output.weight", "offset": 4972732800, "shape": [ 3072, 32064 ], "size": 56623104, "type": 1 }, { "name": "blk.21.attn_norm.weight", "offset": 5169734016, "shape": [ 3072 ], "size": 197001216, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 5169746304, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.21.ffn_up.weight", "offset": 5220077952, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.21.ffn_norm.weight", "offset": 5320741248, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.21.attn_qkv.weight", "offset": 5320753536, "shape": [ 3072, 9216 ], "size": 12288, "type": 1 }, { "name": "blk.22.attn_norm.weight", "offset": 5377376640, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 5377388928, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.22.ffn_up.weight", "offset": 5427720576, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.22.ffn_norm.weight", "offset": 5528383872, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 5528396160, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.22.attn_qkv.weight", "offset": 5547270528, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.23.attn_norm.weight", "offset": 5603893632, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 5603905920, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.23.ffn_up.weight", "offset": 5654237568, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.23.ffn_norm.weight", "offset": 5754900864, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 5754913152, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.23.attn_qkv.weight", "offset": 5773787520, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.24.attn_norm.weight", "offset": 5830410624, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 5830422912, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.24.ffn_up.weight", "offset": 5880754560, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.24.ffn_norm.weight", "offset": 5981417856, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 5981430144, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.24.attn_qkv.weight", "offset": 6000304512, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.25.attn_norm.weight", "offset": 6056927616, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 6056939904, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.25.ffn_up.weight", "offset": 6107271552, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.25.ffn_norm.weight", "offset": 6207934848, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 6207947136, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.25.attn_qkv.weight", "offset": 6226821504, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.26.attn_norm.weight", "offset": 6283444608, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 6283456896, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.26.ffn_up.weight", "offset": 6333788544, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.26.ffn_norm.weight", "offset": 6434451840, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 6434464128, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.26.attn_qkv.weight", "offset": 6453338496, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.27.attn_norm.weight", "offset": 6509961600, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 6509973888, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.27.ffn_up.weight", "offset": 6560305536, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.27.ffn_norm.weight", "offset": 6660968832, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 6660981120, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.27.attn_qkv.weight", "offset": 6679855488, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.28.attn_norm.weight", "offset": 6736478592, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 6736490880, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.28.ffn_up.weight", "offset": 6786822528, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.28.ffn_norm.weight", "offset": 6887485824, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.28.attn_output.weight", "offset": 6887498112, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.28.attn_qkv.weight", "offset": 6906372480, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.29.attn_norm.weight", "offset": 6962995584, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 6963007872, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.29.ffn_up.weight", "offset": 7013339520, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.29.ffn_norm.weight", "offset": 7114002816, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 7114015104, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.29.attn_qkv.weight", "offset": 7132889472, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.30.attn_norm.weight", "offset": 7189512576, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 7189524864, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.30.ffn_up.weight", "offset": 7239856512, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.30.ffn_norm.weight", "offset": 7340519808, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 7340532096, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.30.attn_qkv.weight", "offset": 7359406464, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "blk.31.attn_norm.weight", "offset": 7416029568, "shape": [ 3072 ], "size": 56623104, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 7416041856, "shape": [ 8192, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.31.ffn_up.weight", "offset": 7466373504, "shape": [ 3072, 16384 ], "size": 50331648, "type": 1 }, { "name": "blk.31.ffn_norm.weight", "offset": 7567036800, "shape": [ 3072 ], "size": 100663296, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 7567049088, "shape": [ 3072, 3072 ], "size": 12288, "type": 1 }, { "name": "blk.31.attn_qkv.weight", "offset": 7585923456, "shape": [ 3072, 9216 ], "size": 18874368, "type": 1 }, { "name": "output_norm.weight", "offset": 7642546560, "shape": [ 3072 ], "size": 56623104, "type": 0 } ], "version": 3 }