phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 18, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 134668800, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 134689280, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.ffn_up.weight", "offset": 209953280, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.0.ffn_norm.weight", "offset": 360481280, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 360501760, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.0.attn_qkv.weight", "offset": 382005760, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.1.attn_norm.weight", "offset": 414261760, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 414282240, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.ffn_up.weight", "offset": 489546240, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.1.ffn_norm.weight", "offset": 640074240, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 640094720, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.1.attn_qkv.weight", "offset": 661598720, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.2.attn_norm.weight", "offset": 693854720, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 693875200, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.ffn_up.weight", "offset": 769139200, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.2.ffn_norm.weight", "offset": 919667200, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 919687680, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.2.attn_qkv.weight", "offset": 941191680, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.3.attn_norm.weight", "offset": 973447680, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 973468160, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.ffn_up.weight", "offset": 1048732160, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.3.ffn_norm.weight", "offset": 1199260160, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1199280640, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.3.attn_qkv.weight", "offset": 1220784640, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.4.attn_norm.weight", "offset": 1253040640, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1253061120, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.ffn_up.weight", "offset": 1328325120, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.4.ffn_norm.weight", "offset": 1478853120, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1478873600, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.4.attn_qkv.weight", "offset": 1500377600, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.5.attn_norm.weight", "offset": 1532633600, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1532654080, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.5.ffn_up.weight", "offset": 1607918080, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.5.ffn_norm.weight", "offset": 1758446080, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 1758466560, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.5.attn_qkv.weight", "offset": 1779970560, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.6.ffn_up.weight", "offset": 1812226560, "shape": [ 5120, 35840 ], "size": 32256000, "type": 14 }, { "name": "blk.6.attn_output.weight", "offset": 1962754560, "shape": [ 5120, 5120 ], "size": 150528000, "type": 14 }, { "name": "blk.6.attn_qkv.weight", "offset": 1984258560, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.10.attn_norm.weight", "offset": 2016514560, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2016535040, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.10.ffn_up.weight", "offset": 2091799040, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.10.ffn_norm.weight", "offset": 2242327040, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 2242347520, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.10.attn_qkv.weight", "offset": 2263851520, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.11.attn_norm.weight", "offset": 2296107520, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2296128000, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.ffn_up.weight", "offset": 2371392000, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.11.ffn_norm.weight", "offset": 2521920000, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 2521940480, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.11.attn_qkv.weight", "offset": 2543444480, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.12.attn_norm.weight", "offset": 2575700480, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 2575720960, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.12.ffn_up.weight", "offset": 2650984960, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.12.ffn_norm.weight", "offset": 2801512960, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 2801533440, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.12.attn_qkv.weight", "offset": 2823037440, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.13.attn_norm.weight", "offset": 2855293440, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 2855313920, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.13.ffn_up.weight", "offset": 2930577920, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.13.ffn_norm.weight", "offset": 3081105920, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 3081126400, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.13.attn_qkv.weight", "offset": 3102630400, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.6.attn_norm.weight", "offset": 3134886400, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 3134906880, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.6.ffn_norm.weight", "offset": 3210170880, "shape": [ 5120 ], "size": 75264000, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 3210191360, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 3210211840, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.7.ffn_up.weight", "offset": 3285475840, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.7.ffn_norm.weight", "offset": 3436003840, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 3436024320, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.7.attn_qkv.weight", "offset": 3457528320, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.8.attn_norm.weight", "offset": 3489784320, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 3489804800, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.8.ffn_up.weight", "offset": 3565068800, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.8.ffn_norm.weight", "offset": 3715596800, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 3715617280, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.8.attn_qkv.weight", "offset": 3737121280, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.9.attn_norm.weight", "offset": 3769377280, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 3769397760, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.ffn_up.weight", "offset": 3844661760, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.9.ffn_norm.weight", "offset": 3995189760, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 3995210240, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.9.attn_qkv.weight", "offset": 4016714240, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.14.attn_norm.weight", "offset": 4048970240, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 4048990720, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.14.ffn_up.weight", "offset": 4124254720, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.14.ffn_norm.weight", "offset": 4274782720, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 4274803200, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.14.attn_qkv.weight", "offset": 4296307200, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.15.attn_norm.weight", "offset": 4328563200, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 4328583680, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.15.ffn_up.weight", "offset": 4403847680, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.15.ffn_norm.weight", "offset": 4554375680, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 4554396160, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.15.attn_qkv.weight", "offset": 4575900160, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.16.attn_norm.weight", "offset": 4608156160, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 4608176640, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.ffn_up.weight", "offset": 4683440640, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.16.ffn_norm.weight", "offset": 4833968640, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 4833989120, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.16.attn_qkv.weight", "offset": 4855493120, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.17.attn_norm.weight", "offset": 4887749120, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 4887769600, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.17.ffn_up.weight", "offset": 4963033600, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.17.ffn_norm.weight", "offset": 5113561600, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 5113582080, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.17.attn_qkv.weight", "offset": 5135086080, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.18.attn_norm.weight", "offset": 5167342080, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 5167362560, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.18.ffn_up.weight", "offset": 5242626560, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.18.ffn_norm.weight", "offset": 5393154560, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 5393175040, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.18.attn_qkv.weight", "offset": 5414679040, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.19.attn_norm.weight", "offset": 5446935040, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 5446955520, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.ffn_up.weight", "offset": 5522219520, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.19.ffn_norm.weight", "offset": 5672747520, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 5672768000, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.19.attn_qkv.weight", "offset": 5694272000, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.20.attn_norm.weight", "offset": 5726528000, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 5726548480, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.20.ffn_up.weight", "offset": 5801812480, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.20.ffn_norm.weight", "offset": 5952340480, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 5952360960, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.20.attn_qkv.weight", "offset": 5973864960, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.21.attn_output.weight", "offset": 6006120960, "shape": [ 5120, 5120 ], "size": 32256000, "type": 14 }, { "name": "blk.21.attn_qkv.weight", "offset": 6027624960, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.21.attn_norm.weight", "offset": 6059880960, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 6059901440, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.21.ffn_up.weight", "offset": 6135165440, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.21.ffn_norm.weight", "offset": 6285693440, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 6285713920, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 6285734400, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.ffn_up.weight", "offset": 6360998400, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.22.ffn_norm.weight", "offset": 6511526400, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 6511546880, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.22.attn_qkv.weight", "offset": 6533050880, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.23.attn_norm.weight", "offset": 6565306880, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 6565327360, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.23.ffn_up.weight", "offset": 6640591360, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.23.ffn_norm.weight", "offset": 6791119360, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 6791139840, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.23.attn_qkv.weight", "offset": 6812643840, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.24.attn_norm.weight", "offset": 6844899840, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 6844920320, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.24.ffn_up.weight", "offset": 6920184320, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.24.ffn_norm.weight", "offset": 7070712320, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 7070732800, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.24.attn_qkv.weight", "offset": 7092236800, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.25.attn_norm.weight", "offset": 7124492800, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 7124513280, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.ffn_up.weight", "offset": 7199777280, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.25.ffn_norm.weight", "offset": 7350305280, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 7350325760, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.25.attn_qkv.weight", "offset": 7371829760, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.26.attn_norm.weight", "offset": 7404085760, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 7404106240, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.26.ffn_up.weight", "offset": 7479370240, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.26.ffn_norm.weight", "offset": 7629898240, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 7629918720, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.26.attn_qkv.weight", "offset": 7651422720, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.27.attn_norm.weight", "offset": 7683678720, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 7683699200, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.27.ffn_up.weight", "offset": 7758963200, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.27.ffn_norm.weight", "offset": 7909491200, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 7909511680, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.27.attn_qkv.weight", "offset": 7931015680, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.28.attn_output.weight", "offset": 7963271680, "shape": [ 5120, 5120 ], "size": 32256000, "type": 14 }, { "name": "blk.28.attn_qkv.weight", "offset": 7984775680, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.28.attn_norm.weight", "offset": 8017031680, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 8017052160, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.28.ffn_up.weight", "offset": 8092316160, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.28.ffn_norm.weight", "offset": 8242844160, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 8242864640, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 8242885120, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.29.ffn_up.weight", "offset": 8318149120, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.29.ffn_norm.weight", "offset": 8468677120, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 8468697600, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.29.attn_qkv.weight", "offset": 8490201600, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.30.attn_norm.weight", "offset": 8522457600, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 8522478080, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.30.ffn_up.weight", "offset": 8597742080, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.30.ffn_norm.weight", "offset": 8748270080, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 8748290560, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.30.attn_qkv.weight", "offset": 8769794560, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.31.attn_norm.weight", "offset": 8802050560, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 8802071040, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.31.ffn_up.weight", "offset": 8877335040, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.31.ffn_norm.weight", "offset": 9027863040, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 9027883520, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.31.attn_qkv.weight", "offset": 9049387520, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.32.attn_norm.weight", "offset": 9081643520, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 9081664000, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.32.ffn_up.weight", "offset": 9156928000, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.32.ffn_norm.weight", "offset": 9307456000, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 9307476480, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.32.attn_qkv.weight", "offset": 9328980480, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.33.attn_norm.weight", "offset": 9361236480, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 9361256960, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.33.ffn_up.weight", "offset": 9436520960, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.33.ffn_norm.weight", "offset": 9587048960, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 9587069440, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.33.attn_qkv.weight", "offset": 9608573440, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.34.attn_norm.weight", "offset": 9640829440, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 9640849920, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.ffn_up.weight", "offset": 9716113920, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.34.ffn_norm.weight", "offset": 9866641920, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 9866662400, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.34.attn_qkv.weight", "offset": 9888166400, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.35.attn_output.weight", "offset": 9920422400, "shape": [ 5120, 5120 ], "size": 32256000, "type": 14 }, { "name": "blk.35.attn_qkv.weight", "offset": 9941926400, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "output.weight", "offset": 9974182400, "shape": [ 5120, 32064 ], "size": 32256000, "type": 14 }, { "name": "blk.35.attn_norm.weight", "offset": 10108851200, "shape": [ 5120 ], "size": 134668800, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 10108871680, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.35.ffn_up.weight", "offset": 10184135680, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.35.ffn_norm.weight", "offset": 10334663680, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 10334684160, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 10334704640, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.ffn_up.weight", "offset": 10409968640, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.36.ffn_norm.weight", "offset": 10560496640, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 10560517120, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.36.attn_qkv.weight", "offset": 10582021120, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.37.attn_norm.weight", "offset": 10614277120, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 10614297600, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.ffn_up.weight", "offset": 10689561600, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.37.ffn_norm.weight", "offset": 10840089600, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 10840110080, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.37.attn_qkv.weight", "offset": 10861614080, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.38.attn_norm.weight", "offset": 10893870080, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 10893890560, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.ffn_up.weight", "offset": 10969154560, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.38.ffn_norm.weight", "offset": 11119682560, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 11119703040, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.38.attn_qkv.weight", "offset": 11141207040, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "blk.39.attn_norm.weight", "offset": 11173463040, "shape": [ 5120 ], "size": 32256000, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 11173483520, "shape": [ 17920, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.ffn_up.weight", "offset": 11248747520, "shape": [ 5120, 35840 ], "size": 75264000, "type": 14 }, { "name": "blk.39.ffn_norm.weight", "offset": 11399275520, "shape": [ 5120 ], "size": 150528000, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 11399296000, "shape": [ 5120, 5120 ], "size": 20480, "type": 14 }, { "name": "blk.39.attn_qkv.weight", "offset": 11420800000, "shape": [ 5120, 7680 ], "size": 21504000, "type": 14 }, { "name": "output_norm.weight", "offset": 11453056000, "shape": [ 5120 ], "size": 32256000, "type": 0 } ], "version": 3 }