phi3

{ "metadata": { "general.architecture": "phi3", "general.file_type": 7, "general.name": "Phi3", "general.quantization_version": 2, "phi3.attention.head_count": 40, "phi3.attention.head_count_kv": 10, "phi3.attention.layer_norm_rms_epsilon": 0.00001, "phi3.block_count": 40, "phi3.context_length": 4096, "phi3.embedding_length": 5120, "phi3.feed_forward_length": 17920, "phi3.rope.dimension_count": 128, "phi3.rope.freq_base": 10000, "phi3.rope.scaling.original_context_length": 4096, "tokenizer.ggml.add_bos_token": false, "tokenizer.ggml.add_eos_token": false, "tokenizer.ggml.bos_token_id": 1, "tokenizer.ggml.eos_token_id": 32000, "tokenizer.ggml.model": "llama", "tokenizer.ggml.padding_token_id": 32000, "tokenizer.ggml.pre": "default", "tokenizer.ggml.scores": "... (32064 values)", "tokenizer.ggml.token_type": "... (32064 values)", "tokenizer.ggml.tokens": "... (32064 values)", "tokenizer.ggml.unknown_token_id": 0 }, "num_params": 13960238080, "tensors": [ { "name": "blk.0.attn_norm.weight", "offset": 174428160, "shape": [ 5120 ], "size": 174428160, "type": 0 }, { "name": "blk.0.ffn_down.weight", "offset": 174448640, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.0.ffn_up.weight", "offset": 271933440, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.0.ffn_norm.weight", "offset": 466903040, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.0.attn_output.weight", "offset": 466923520, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.0.attn_qkv.weight", "offset": 494776320, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.1.attn_norm.weight", "offset": 536555520, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.1.ffn_down.weight", "offset": 536576000, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.1.ffn_up.weight", "offset": 634060800, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.1.ffn_norm.weight", "offset": 829030400, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.1.attn_output.weight", "offset": 829050880, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.1.attn_qkv.weight", "offset": 856903680, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.2.attn_norm.weight", "offset": 898682880, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.2.ffn_down.weight", "offset": 898703360, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.2.ffn_up.weight", "offset": 996188160, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.2.ffn_norm.weight", "offset": 1191157760, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.2.attn_output.weight", "offset": 1191178240, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.2.attn_qkv.weight", "offset": 1219031040, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.3.attn_norm.weight", "offset": 1260810240, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.3.ffn_down.weight", "offset": 1260830720, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.3.ffn_up.weight", "offset": 1358315520, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.3.ffn_norm.weight", "offset": 1553285120, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.3.attn_output.weight", "offset": 1553305600, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.3.attn_qkv.weight", "offset": 1581158400, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.4.attn_norm.weight", "offset": 1622937600, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.4.ffn_down.weight", "offset": 1622958080, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.4.ffn_up.weight", "offset": 1720442880, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.4.ffn_norm.weight", "offset": 1915412480, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.4.attn_output.weight", "offset": 1915432960, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.4.attn_qkv.weight", "offset": 1943285760, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.5.attn_norm.weight", "offset": 1985064960, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.5.ffn_down.weight", "offset": 1985085440, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.5.ffn_up.weight", "offset": 2082570240, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.5.ffn_norm.weight", "offset": 2277539840, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.5.attn_output.weight", "offset": 2277560320, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.5.attn_qkv.weight", "offset": 2305413120, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.6.ffn_up.weight", "offset": 2347192320, "shape": [ 5120, 35840 ], "size": 41779200, "type": 8 }, { "name": "blk.6.attn_output.weight", "offset": 2542161920, "shape": [ 5120, 5120 ], "size": 194969600, "type": 8 }, { "name": "blk.6.attn_qkv.weight", "offset": 2570014720, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.10.attn_norm.weight", "offset": 2611793920, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.10.ffn_down.weight", "offset": 2611814400, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.10.ffn_up.weight", "offset": 2709299200, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.10.ffn_norm.weight", "offset": 2904268800, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.10.attn_output.weight", "offset": 2904289280, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.10.attn_qkv.weight", "offset": 2932142080, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.11.attn_norm.weight", "offset": 2973921280, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.11.ffn_down.weight", "offset": 2973941760, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.11.ffn_up.weight", "offset": 3071426560, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.11.ffn_norm.weight", "offset": 3266396160, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.11.attn_output.weight", "offset": 3266416640, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.11.attn_qkv.weight", "offset": 3294269440, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.12.attn_norm.weight", "offset": 3336048640, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.12.ffn_down.weight", "offset": 3336069120, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.12.ffn_up.weight", "offset": 3433553920, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.12.ffn_norm.weight", "offset": 3628523520, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.12.attn_output.weight", "offset": 3628544000, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.12.attn_qkv.weight", "offset": 3656396800, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.13.attn_norm.weight", "offset": 3698176000, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.13.ffn_down.weight", "offset": 3698196480, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.13.ffn_up.weight", "offset": 3795681280, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.13.ffn_norm.weight", "offset": 3990650880, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.13.attn_output.weight", "offset": 3990671360, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.13.attn_qkv.weight", "offset": 4018524160, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.6.attn_norm.weight", "offset": 4060303360, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.6.ffn_down.weight", "offset": 4060323840, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.6.ffn_norm.weight", "offset": 4157808640, "shape": [ 5120 ], "size": 97484800, "type": 0 }, { "name": "blk.7.attn_norm.weight", "offset": 4157829120, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.7.ffn_down.weight", "offset": 4157849600, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.7.ffn_up.weight", "offset": 4255334400, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.7.ffn_norm.weight", "offset": 4450304000, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.7.attn_output.weight", "offset": 4450324480, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.7.attn_qkv.weight", "offset": 4478177280, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.8.attn_norm.weight", "offset": 4519956480, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.8.ffn_down.weight", "offset": 4519976960, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.8.ffn_up.weight", "offset": 4617461760, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.8.ffn_norm.weight", "offset": 4812431360, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.8.attn_output.weight", "offset": 4812451840, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.8.attn_qkv.weight", "offset": 4840304640, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.9.attn_norm.weight", "offset": 4882083840, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.9.ffn_down.weight", "offset": 4882104320, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.9.ffn_up.weight", "offset": 4979589120, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.9.ffn_norm.weight", "offset": 5174558720, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.9.attn_output.weight", "offset": 5174579200, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.9.attn_qkv.weight", "offset": 5202432000, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.14.attn_norm.weight", "offset": 5244211200, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.14.ffn_down.weight", "offset": 5244231680, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.14.ffn_up.weight", "offset": 5341716480, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.14.ffn_norm.weight", "offset": 5536686080, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.14.attn_output.weight", "offset": 5536706560, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.14.attn_qkv.weight", "offset": 5564559360, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.15.attn_norm.weight", "offset": 5606338560, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.15.ffn_down.weight", "offset": 5606359040, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.15.ffn_up.weight", "offset": 5703843840, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.15.ffn_norm.weight", "offset": 5898813440, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.15.attn_output.weight", "offset": 5898833920, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.15.attn_qkv.weight", "offset": 5926686720, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.16.attn_norm.weight", "offset": 5968465920, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.16.ffn_down.weight", "offset": 5968486400, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.16.ffn_up.weight", "offset": 6065971200, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.16.ffn_norm.weight", "offset": 6260940800, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.16.attn_output.weight", "offset": 6260961280, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.16.attn_qkv.weight", "offset": 6288814080, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.17.attn_norm.weight", "offset": 6330593280, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.17.ffn_down.weight", "offset": 6330613760, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.17.ffn_up.weight", "offset": 6428098560, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.17.ffn_norm.weight", "offset": 6623068160, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.17.attn_output.weight", "offset": 6623088640, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.17.attn_qkv.weight", "offset": 6650941440, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.18.attn_norm.weight", "offset": 6692720640, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.18.ffn_down.weight", "offset": 6692741120, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.18.ffn_up.weight", "offset": 6790225920, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.18.ffn_norm.weight", "offset": 6985195520, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.18.attn_output.weight", "offset": 6985216000, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.18.attn_qkv.weight", "offset": 7013068800, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.19.attn_norm.weight", "offset": 7054848000, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.19.ffn_down.weight", "offset": 7054868480, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.19.ffn_up.weight", "offset": 7152353280, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.19.ffn_norm.weight", "offset": 7347322880, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.19.attn_output.weight", "offset": 7347343360, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.19.attn_qkv.weight", "offset": 7375196160, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.20.attn_norm.weight", "offset": 7416975360, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.20.ffn_down.weight", "offset": 7416995840, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.20.ffn_up.weight", "offset": 7514480640, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.20.ffn_norm.weight", "offset": 7709450240, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.20.attn_output.weight", "offset": 7709470720, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.20.attn_qkv.weight", "offset": 7737323520, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.21.attn_output.weight", "offset": 7779102720, "shape": [ 5120, 5120 ], "size": 41779200, "type": 8 }, { "name": "blk.21.attn_qkv.weight", "offset": 7806955520, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.21.attn_norm.weight", "offset": 7848734720, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.21.ffn_down.weight", "offset": 7848755200, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.21.ffn_up.weight", "offset": 7946240000, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.21.ffn_norm.weight", "offset": 8141209600, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.22.attn_norm.weight", "offset": 8141230080, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.22.ffn_down.weight", "offset": 8141250560, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.22.ffn_up.weight", "offset": 8238735360, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.22.ffn_norm.weight", "offset": 8433704960, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.22.attn_output.weight", "offset": 8433725440, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.22.attn_qkv.weight", "offset": 8461578240, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.23.attn_norm.weight", "offset": 8503357440, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.23.ffn_down.weight", "offset": 8503377920, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.23.ffn_up.weight", "offset": 8600862720, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.23.ffn_norm.weight", "offset": 8795832320, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.23.attn_output.weight", "offset": 8795852800, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.23.attn_qkv.weight", "offset": 8823705600, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.24.attn_norm.weight", "offset": 8865484800, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.24.ffn_down.weight", "offset": 8865505280, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.24.ffn_up.weight", "offset": 8962990080, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.24.ffn_norm.weight", "offset": 9157959680, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.24.attn_output.weight", "offset": 9157980160, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.24.attn_qkv.weight", "offset": 9185832960, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.25.attn_norm.weight", "offset": 9227612160, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.25.ffn_down.weight", "offset": 9227632640, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.25.ffn_up.weight", "offset": 9325117440, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.25.ffn_norm.weight", "offset": 9520087040, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.25.attn_output.weight", "offset": 9520107520, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.25.attn_qkv.weight", "offset": 9547960320, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.26.attn_norm.weight", "offset": 9589739520, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.26.ffn_down.weight", "offset": 9589760000, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.26.ffn_up.weight", "offset": 9687244800, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.26.ffn_norm.weight", "offset": 9882214400, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.26.attn_output.weight", "offset": 9882234880, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.26.attn_qkv.weight", "offset": 9910087680, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.27.attn_norm.weight", "offset": 9951866880, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.27.ffn_down.weight", "offset": 9951887360, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.27.ffn_up.weight", "offset": 10049372160, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.27.ffn_norm.weight", "offset": 10244341760, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.27.attn_output.weight", "offset": 10244362240, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.27.attn_qkv.weight", "offset": 10272215040, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.28.attn_output.weight", "offset": 10313994240, "shape": [ 5120, 5120 ], "size": 41779200, "type": 8 }, { "name": "blk.28.attn_qkv.weight", "offset": 10341847040, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.28.attn_norm.weight", "offset": 10383626240, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.28.ffn_down.weight", "offset": 10383646720, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.28.ffn_up.weight", "offset": 10481131520, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.28.ffn_norm.weight", "offset": 10676101120, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.29.attn_norm.weight", "offset": 10676121600, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.29.ffn_down.weight", "offset": 10676142080, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.29.ffn_up.weight", "offset": 10773626880, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.29.ffn_norm.weight", "offset": 10968596480, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.29.attn_output.weight", "offset": 10968616960, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.29.attn_qkv.weight", "offset": 10996469760, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.30.attn_norm.weight", "offset": 11038248960, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.30.ffn_down.weight", "offset": 11038269440, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.30.ffn_up.weight", "offset": 11135754240, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.30.ffn_norm.weight", "offset": 11330723840, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.30.attn_output.weight", "offset": 11330744320, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.30.attn_qkv.weight", "offset": 11358597120, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.31.attn_norm.weight", "offset": 11400376320, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.31.ffn_down.weight", "offset": 11400396800, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.31.ffn_up.weight", "offset": 11497881600, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.31.ffn_norm.weight", "offset": 11692851200, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.31.attn_output.weight", "offset": 11692871680, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.31.attn_qkv.weight", "offset": 11720724480, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.32.attn_norm.weight", "offset": 11762503680, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.32.ffn_down.weight", "offset": 11762524160, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.32.ffn_up.weight", "offset": 11860008960, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.32.ffn_norm.weight", "offset": 12054978560, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.32.attn_output.weight", "offset": 12054999040, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.32.attn_qkv.weight", "offset": 12082851840, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.33.attn_norm.weight", "offset": 12124631040, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.33.ffn_down.weight", "offset": 12124651520, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.33.ffn_up.weight", "offset": 12222136320, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.33.ffn_norm.weight", "offset": 12417105920, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.33.attn_output.weight", "offset": 12417126400, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.33.attn_qkv.weight", "offset": 12444979200, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.34.attn_norm.weight", "offset": 12486758400, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.34.ffn_down.weight", "offset": 12486778880, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.34.ffn_up.weight", "offset": 12584263680, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.34.ffn_norm.weight", "offset": 12779233280, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.34.attn_output.weight", "offset": 12779253760, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.34.attn_qkv.weight", "offset": 12807106560, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.35.attn_output.weight", "offset": 12848885760, "shape": [ 5120, 5120 ], "size": 41779200, "type": 8 }, { "name": "blk.35.attn_qkv.weight", "offset": 12876738560, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "output.weight", "offset": 12918517760, "shape": [ 5120, 32064 ], "size": 41779200, "type": 8 }, { "name": "blk.35.attn_norm.weight", "offset": 13092945920, "shape": [ 5120 ], "size": 174428160, "type": 0 }, { "name": "blk.35.ffn_down.weight", "offset": 13092966400, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.35.ffn_up.weight", "offset": 13190451200, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.35.ffn_norm.weight", "offset": 13385420800, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.36.attn_norm.weight", "offset": 13385441280, "shape": [ 5120 ], "size": 20480, "type": 0 }, { "name": "blk.36.ffn_down.weight", "offset": 13385461760, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.36.ffn_up.weight", "offset": 13482946560, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.36.ffn_norm.weight", "offset": 13677916160, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.36.attn_output.weight", "offset": 13677936640, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.36.attn_qkv.weight", "offset": 13705789440, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.37.attn_norm.weight", "offset": 13747568640, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.37.ffn_down.weight", "offset": 13747589120, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.37.ffn_up.weight", "offset": 13845073920, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.37.ffn_norm.weight", "offset": 14040043520, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.37.attn_output.weight", "offset": 14040064000, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.37.attn_qkv.weight", "offset": 14067916800, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.38.attn_norm.weight", "offset": 14109696000, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.38.ffn_down.weight", "offset": 14109716480, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.38.ffn_up.weight", "offset": 14207201280, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.38.ffn_norm.weight", "offset": 14402170880, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.38.attn_output.weight", "offset": 14402191360, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.38.attn_qkv.weight", "offset": 14430044160, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "blk.39.attn_norm.weight", "offset": 14471823360, "shape": [ 5120 ], "size": 41779200, "type": 0 }, { "name": "blk.39.ffn_down.weight", "offset": 14471843840, "shape": [ 17920, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.39.ffn_up.weight", "offset": 14569328640, "shape": [ 5120, 35840 ], "size": 97484800, "type": 8 }, { "name": "blk.39.ffn_norm.weight", "offset": 14764298240, "shape": [ 5120 ], "size": 194969600, "type": 0 }, { "name": "blk.39.attn_output.weight", "offset": 14764318720, "shape": [ 5120, 5120 ], "size": 20480, "type": 8 }, { "name": "blk.39.attn_qkv.weight", "offset": 14792171520, "shape": [ 5120, 7680 ], "size": 27852800, "type": 8 }, { "name": "output_norm.weight", "offset": 14833950720, "shape": [ 5120 ], "size": 41779200, "type": 0 } ], "version": 3 }