phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 2, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 92344320, "shape": [ 5120 ], "size": 92344320, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 92364800, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.0.ffn_up.weight", "offset": 143974400, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.0.ffn_norm.weight", "offset": 247193600, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 247214080, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.0.attn_qkv.weight", "offset": 261959680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.1.attn_norm.weight", "offset": 284078080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 284098560, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.1.ffn_up.weight", "offset": 335708160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.1.ffn_norm.weight", "offset": 438927360, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 438947840, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.1.attn_qkv.weight", "offset": 453693440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.2.attn_norm.weight", "offset": 475811840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 475832320, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.2.ffn_up.weight", "offset": 527441920, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.2.ffn_norm.weight", "offset": 630661120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 630681600, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.2.attn_qkv.weight", "offset": 645427200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.3.attn_norm.weight", "offset": 667545600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 667566080, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.3.ffn_up.weight", "offset": 719175680, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.3.ffn_norm.weight", "offset": 822394880, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 822415360, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.3.attn_qkv.weight", "offset": 837160960, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.4.attn_norm.weight", "offset": 859279360, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 859299840, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.4.ffn_up.weight", "offset": 910909440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.4.ffn_norm.weight", "offset": 1014128640, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1014149120, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.4.attn_qkv.weight", "offset": 1028894720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.5.attn_norm.weight", "offset": 1051013120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1051033600, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.5.ffn_up.weight", "offset": 1102643200, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.5.ffn_norm.weight", "offset": 1205862400, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1205882880, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.5.attn_qkv.weight", "offset": 1220628480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.6.ffn_up.weight", "offset": 1242746880, "shape": [ 5120, 35840 ], "size": 22118400, "type": 2 }, { "name": "blk.6.attn_output.weight", "offset": 1345966080, "shape": [ 5120, 5120 ], "size": 103219200, "type": 2 }, { "name": "blk.6.attn_qkv.weight", "offset": 1360711680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.10.attn_norm.weight", "offset": 1382830080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 1382850560, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.10.ffn_up.weight", "offset": 1434460160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.10.ffn_norm.weight", "offset": 1537679360, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 1537699840, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.10.attn_qkv.weight", "offset": 1552445440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.11.attn_norm.weight", "offset": 1574563840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 1574584320, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.11.ffn_up.weight", "offset": 1626193920, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.11.ffn_norm.weight", "offset": 1729413120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 1729433600, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.11.attn_qkv.weight", "offset": 1744179200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.12.attn_norm.weight", "offset": 1766297600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 1766318080, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.12.ffn_up.weight", "offset": 1817927680, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.12.ffn_norm.weight", "offset": 1921146880, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 1921167360, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.12.attn_qkv.weight", "offset": 1935912960, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.13.attn_norm.weight", "offset": 1958031360, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 1958051840, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.13.ffn_up.weight", "offset": 2009661440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.13.ffn_norm.weight", "offset": 2112880640, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 2112901120, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.13.attn_qkv.weight", "offset": 2127646720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.6.attn_norm.weight", "offset": 2149765120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 2149785600, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.6.ffn_norm.weight", "offset": 2201395200, "shape": [ 5120 ], "size": 51609600, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 2201415680, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 2201436160, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.7.ffn_up.weight", "offset": 2253045760, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.7.ffn_norm.weight", "offset": 2356264960, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 2356285440, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.7.attn_qkv.weight", "offset": 2371031040, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.8.attn_norm.weight", "offset": 2393149440, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 2393169920, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.8.ffn_up.weight", "offset": 2444779520, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.8.ffn_norm.weight", "offset": 2547998720, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 2548019200, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.8.attn_qkv.weight", "offset": 2562764800, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.9.attn_norm.weight", "offset": 2584883200, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 2584903680, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.9.ffn_up.weight", "offset": 2636513280, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.9.ffn_norm.weight", "offset": 2739732480, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 2739752960, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.9.attn_qkv.weight", "offset": 2754498560, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.14.attn_norm.weight", "offset": 2776616960, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 2776637440, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.14.ffn_up.weight", "offset": 2828247040, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.14.ffn_norm.weight", "offset": 2931466240, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 2931486720, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.14.attn_qkv.weight", "offset": 2946232320, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.15.attn_norm.weight", "offset": 2968350720, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 2968371200, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.15.ffn_up.weight", "offset": 3019980800, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.15.ffn_norm.weight", "offset": 3123200000, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 3123220480, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.15.attn_qkv.weight", "offset": 3137966080, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.16.attn_norm.weight", "offset": 3160084480, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 3160104960, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.16.ffn_up.weight", "offset": 3211714560, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.16.ffn_norm.weight", "offset": 3314933760, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 3314954240, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.16.attn_qkv.weight", "offset": 3329699840, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.17.attn_norm.weight", "offset": 3351818240, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 3351838720, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.17.ffn_up.weight", "offset": 3403448320, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.17.ffn_norm.weight", "offset": 3506667520, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 3506688000, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.17.attn_qkv.weight", "offset": 3521433600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.18.attn_norm.weight", "offset": 3543552000, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 3543572480, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.18.ffn_up.weight", "offset": 3595182080, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.18.ffn_norm.weight", "offset": 3698401280, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 3698421760, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.18.attn_qkv.weight", "offset": 3713167360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.19.attn_norm.weight", "offset": 3735285760, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 3735306240, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.19.ffn_up.weight", "offset": 3786915840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.19.ffn_norm.weight", "offset": 3890135040, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 3890155520, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.19.attn_qkv.weight", "offset": 3904901120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.20.attn_norm.weight", "offset": 3927019520, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 3927040000, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.20.ffn_up.weight", "offset": 3978649600, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.20.ffn_norm.weight", "offset": 4081868800, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 4081889280, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.20.attn_qkv.weight", "offset": 4096634880, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.21.attn_output.weight", "offset": 4118753280, "shape": [ 5120, 5120 ], "size": 22118400, "type": 2 }, { "name": "blk.21.attn_qkv.weight", "offset": 4133498880, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.21.attn_norm.weight", "offset": 4155617280, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 4155637760, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.21.ffn_up.weight", "offset": 4207247360, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.21.ffn_norm.weight", "offset": 4310466560, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 4310487040, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 4310507520, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.22.ffn_up.weight", "offset": 4362117120, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.22.ffn_norm.weight", "offset": 4465336320, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 4465356800, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.22.attn_qkv.weight", "offset": 4480102400, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.23.attn_norm.weight", "offset": 4502220800, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 4502241280, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.23.ffn_up.weight", "offset": 4553850880, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.23.ffn_norm.weight", "offset": 4657070080, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 4657090560, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.23.attn_qkv.weight", "offset": 4671836160, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.24.attn_norm.weight", "offset": 4693954560, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 4693975040, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.24.ffn_up.weight", "offset": 4745584640, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.24.ffn_norm.weight", "offset": 4848803840, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 4848824320, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.24.attn_qkv.weight", "offset": 4863569920, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.25.attn_norm.weight", "offset": 4885688320, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 4885708800, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.25.ffn_up.weight", "offset": 4937318400, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.25.ffn_norm.weight", "offset": 5040537600, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 5040558080, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.25.attn_qkv.weight", "offset": 5055303680, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.26.attn_norm.weight", "offset": 5077422080, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 5077442560, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.26.ffn_up.weight", "offset": 5129052160, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.26.ffn_norm.weight", "offset": 5232271360, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 5232291840, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.26.attn_qkv.weight", "offset": 5247037440, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.27.attn_norm.weight", "offset": 5269155840, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 5269176320, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.27.ffn_up.weight", "offset": 5320785920, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.27.ffn_norm.weight", "offset": 5424005120, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 5424025600, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.27.attn_qkv.weight", "offset": 5438771200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.28.attn_output.weight", "offset": 5460889600, "shape": [ 5120, 5120 ], "size": 22118400, "type": 2 }, { "name": "blk.28.attn_qkv.weight", "offset": 5475635200, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.28.attn_norm.weight", "offset": 5497753600, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 5497774080, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.28.ffn_up.weight", "offset": 5549383680, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.28.ffn_norm.weight", "offset": 5652602880, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 5652623360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 5652643840, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.29.ffn_up.weight", "offset": 5704253440, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.29.ffn_norm.weight", "offset": 5807472640, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 5807493120, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.29.attn_qkv.weight", "offset": 5822238720, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.30.attn_norm.weight", "offset": 5844357120, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 5844377600, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.30.ffn_up.weight", "offset": 5895987200, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.30.ffn_norm.weight", "offset": 5999206400, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 5999226880, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.30.attn_qkv.weight", "offset": 6013972480, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.31.attn_norm.weight", "offset": 6036090880, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 6036111360, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.31.ffn_up.weight", "offset": 6087720960, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.31.ffn_norm.weight", "offset": 6190940160, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 6190960640, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.31.attn_qkv.weight", "offset": 6205706240, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.32.attn_norm.weight", "offset": 6227824640, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 6227845120, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.32.ffn_up.weight", "offset": 6279454720, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.32.ffn_norm.weight", "offset": 6382673920, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 6382694400, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.32.attn_qkv.weight", "offset": 6397440000, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.33.attn_norm.weight", "offset": 6419558400, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 6419578880, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.33.ffn_up.weight", "offset": 6471188480, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.33.ffn_norm.weight", "offset": 6574407680, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 6574428160, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.33.attn_qkv.weight", "offset": 6589173760, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.34.attn_norm.weight", "offset": 6611292160, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 6611312640, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.34.ffn_up.weight", "offset": 6662922240, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.34.ffn_norm.weight", "offset": 6766141440, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 6766161920, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.34.attn_qkv.weight", "offset": 6780907520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.35.attn_output.weight", "offset": 6803025920, "shape": [ 5120, 5120 ], "size": 22118400, "type": 2 }, { "name": "blk.35.attn_qkv.weight", "offset": 6817771520, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "output.weight", "offset": 6839889920, "shape": [ 5120, 32064 ], "size": 22118400, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 6974558720, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 6974579200, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.35.ffn_up.weight", "offset": 7026188800, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.35.ffn_norm.weight", "offset": 7129408000, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 7129428480, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 7129448960, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.36.ffn_up.weight", "offset": 7181058560, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.36.ffn_norm.weight", "offset": 7284277760, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 7284298240, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.36.attn_qkv.weight", "offset": 7299043840, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.37.attn_norm.weight", "offset": 7321162240, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 7321182720, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.37.ffn_up.weight", "offset": 7372792320, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.37.ffn_norm.weight", "offset": 7476011520, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 7476032000, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.37.attn_qkv.weight", "offset": 7490777600, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.38.attn_norm.weight", "offset": 7512896000, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 7512916480, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.38.ffn_up.weight", "offset": 7564526080, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.38.ffn_norm.weight", "offset": 7667745280, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 7667765760, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.38.attn_qkv.weight", "offset": 7682511360, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "blk.39.attn_norm.weight", "offset": 7704629760, "shape": [ 5120 ], "size": 22118400, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 7704650240, "shape": [ 17920, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.39.ffn_up.weight", "offset": 7756259840, "shape": [ 5120, 35840 ], "size": 51609600, "type": 2 }, { "name": "blk.39.ffn_norm.weight", "offset": 7859479040, "shape": [ 5120 ], "size": 103219200, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 7859499520, "shape": [ 5120, 5120 ], "size": 20480, "type": 2 }, { "name": "blk.39.attn_qkv.weight", "offset": 7874245120, "shape": [ 5120, 7680 ], "size": 14745600, "type": 2 }, { "name": "output_norm.weight", "offset": 7896363520, "shape": [ 5120 ], "size": 22118400, "type": 0 } ], "version": 3 }