128K_Q8_0
14B
99 Pulls Updated 6 weeks ago
25816d2b932c · 15GB
{
"metadata": {
"general.architecture": "phi3",
"general.file_type": 7,
"general.name": "Phi3",
"general.quantization_version": 2,
"phi3.attention.head_count": 40,
"phi3.attention.head_count_kv": 10,
"phi3.attention.layer_norm_rms_epsilon": 0.00001,
"phi3.block_count": 40,
"phi3.context_length": 4096,
"phi3.embedding_length": 5120,
"phi3.feed_forward_length": 17920,
"phi3.rope.dimension_count": 128,
"phi3.rope.freq_base": 10000,
"phi3.rope.scaling.original_context_length": 4096,
"quantize.imatrix.chunks_count": 234,
"quantize.imatrix.dataset": "/training_data/calibration_data.txt",
"quantize.imatrix.entries_count": 160,
"quantize.imatrix.file": "/models/Phi-3-medium-4k-instruct-GGUF/Phi-3-medium-4k-instruct.imatrix",
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 32000,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 32000,
"tokenizer.ggml.pre": "default",
"tokenizer.ggml.scores": "... (32064 values)",
"tokenizer.ggml.token_type": "... (32064 values)",
"tokenizer.ggml.tokens": "... (32064 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 13960238080,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 174428160,
"shape": [
5120
],
"size": 174428160,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 174448640,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.0.ffn_up.weight",
"offset": 271933440,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 466903040,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.0.attn_output.weight",
"offset": 466923520,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.0.attn_qkv.weight",
"offset": 494776320,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.1.attn_norm.weight",
"offset": 536555520,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 536576000,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.1.ffn_up.weight",
"offset": 634060800,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 829030400,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.1.attn_output.weight",
"offset": 829050880,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.1.attn_qkv.weight",
"offset": 856903680,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.2.attn_norm.weight",
"offset": 898682880,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 898703360,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.2.ffn_up.weight",
"offset": 996188160,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1191157760,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.2.attn_output.weight",
"offset": 1191178240,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.2.attn_qkv.weight",
"offset": 1219031040,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1260810240,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1260830720,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1358315520,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1553285120,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.3.attn_output.weight",
"offset": 1553305600,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.3.attn_qkv.weight",
"offset": 1581158400,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1622937600,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1622958080,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1720442880,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1915412480,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.4.attn_output.weight",
"offset": 1915432960,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.4.attn_qkv.weight",
"offset": 1943285760,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1985064960,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1985085440,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.5.ffn_up.weight",
"offset": 2082570240,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 2277539840,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.5.attn_output.weight",
"offset": 2277560320,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.5.attn_qkv.weight",
"offset": 2305413120,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2347192320,
"shape": [
5120,
35840
],
"size": 41779200,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 2542161920,
"shape": [
5120,
5120
],
"size": 194969600,
"type": 8
},
{
"name": "blk.6.attn_qkv.weight",
"offset": 2570014720,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2611793920,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2611814400,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.10.ffn_up.weight",
"offset": 2709299200,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 2904268800,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.10.attn_output.weight",
"offset": 2904289280,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.10.attn_qkv.weight",
"offset": 2932142080,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.11.attn_norm.weight",
"offset": 2973921280,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "\u0000\u0000\u0019\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.30.attn_",
"offset": 2973941760,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.11.ffn_up.weight",
"offset": 3071426560,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 3266396160,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.11.attn_output.weight",
"offset": 3266416640,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.11.attn_qkv.weight",
"offset": 3294269440,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.12.attn_norm.weight",
"offset": 3336048640,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 3336069120,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.12.ffn_up.weight",
"offset": 3433553920,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 3628523520,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.12.attn_output.weight",
"offset": 3628544000,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.12.attn_qkv.weight",
"offset": 3656396800,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.13.attn_norm.weight",
"offset": 3698176000,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 3698196480,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.13.ffn_up.weight",
"offset": 3795681280,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 3990650880,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.13.attn_output.weight",
"offset": 3990671360,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.13.attn_qkv.weight",
"offset": 4018524160,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.6.attn_norm.weight",
"offset": 4060303360,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 4060323840,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 4157808640,
"shape": [
5120
],
"size": 97484800,
"type": 0
},
{
"name": "blk.7.attn_norm.weight",
"offset": 4157829120,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 4157849600,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.7.ffn_up.weight",
"offset": 4255334400,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 4450304000,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.7.attn_output.weight",
"offset": 4450324480,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.7.attn_qkv.weight",
"offset": 4478177280,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.8.attn_norm.weight",
"offset": 4519956480,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 4519976960,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.8.ffn_up.weight",
"offset": 4617461760,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 4812431360,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.8.attn_output.weight",
"offset": 4812451840,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.8.attn_qkv.weight",
"offset": 4840304640,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.9.attn_norm.weight",
"offset": 4882083840,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 4882104320,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.9.ffn_up.weight",
"offset": 4979589120,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 5174558720,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.9.attn_output.weight",
"offset": 5174579200,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.9.attn_qkv.weight",
"offset": 5202432000,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.14.attn_norm.weight",
"offset": 5244211200,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 5244231680,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.14.ffn_up.weight",
"offset": 5341716480,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 5536686080,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.14.attn_output.weight",
"offset": 5536706560,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.14.attn_qkv.weight",
"offset": 5564559360,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.15.attn_norm.weight",
"offset": 5606338560,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 5606359040,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.15.ffn_up.weight",
"offset": 5703843840,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 5898813440,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.15.attn_output.weight",
"offset": 5898833920,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.15.attn_qkv.weight",
"offset": 5926686720,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.16.attn_norm.weight",
"offset": 5968465920,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 5968486400,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.16.ffn_up.weight",
"offset": 6065971200,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 6260940800,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.16.attn_output.weight",
"offset": 6260961280,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.16.attn_qkv.weight",
"offset": 6288814080,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.17.attn_norm.weight",
"offset": 6330593280,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 6330613760,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.17.ffn_up.weight",
"offset": 6428098560,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 6623068160,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.17.attn_output.weight",
"offset": 6623088640,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.17.attn_qkv.weight",
"offset": 6650941440,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.18.attn_norm.weight",
"offset": 6692720640,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 6692741120,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.18.ffn_up.weight",
"offset": 6790225920,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 6985195520,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.18.attn_output.weight",
"offset": 6985216000,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.18.attn_qkv.weight",
"offset": 7013068800,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.19.attn_norm.weight",
"offset": 7054848000,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 7054868480,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.19.ffn_up.weight",
"offset": 7152353280,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 7347322880,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.19.attn_output.weight",
"offset": 7347343360,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.19.attn_qkv.weight",
"offset": 7375196160,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.20.attn_norm.weight",
"offset": 7416975360,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 7416995840,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.20.ffn_up.weight",
"offset": 7514480640,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 7709450240,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.20.attn_output.weight",
"offset": 7709470720,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.20.attn_qkv.weight",
"offset": 7737323520,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 7779102720,
"shape": [
5120,
5120
],
"size": 41779200,
"type": 8
},
{
"name": "blk.21.attn_qkv.weight",
"offset": 7806955520,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.21.attn_norm.weight",
"offset": 7848734720,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 7848755200,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.21.ffn_up.weight",
"offset": 7946240000,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 8141209600,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.22.attn_norm.weight",
"offset": 8141230080,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 8141250560,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.22.ffn_up.weight",
"offset": 8238735360,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 8433704960,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.22.attn_output.weight",
"offset": 8433725440,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.22.attn_qkv.weight",
"offset": 8461578240,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.23.attn_norm.weight",
"offset": 8503357440,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 8503377920,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.23.ffn_up.weight",
"offset": 8600862720,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 8795832320,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.23.attn_output.weight",
"offset": 8795852800,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.23.attn_qkv.weight",
"offset": 8823705600,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.24.attn_norm.weight",
"offset": 8865484800,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 8865505280,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.24.ffn_up.weight",
"offset": 8962990080,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 9157959680,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.24.attn_output.weight",
"offset": 9157980160,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.24.attn_qkv.weight",
"offset": 9185832960,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.25.attn_norm.weight",
"offset": 9227612160,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 9227632640,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.25.ffn_up.weight",
"offset": 9325117440,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 9520087040,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.25.attn_output.weight",
"offset": 9520107520,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.25.attn_qkv.weight",
"offset": 9547960320,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.26.attn_norm.weight",
"offset": 9589739520,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 9589760000,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.26.ffn_up.weight",
"offset": 9687244800,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 9882214400,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.26.attn_output.weight",
"offset": 9882234880,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.26.attn_qkv.weight",
"offset": 9910087680,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.27.attn_norm.weight",
"offset": 9951866880,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 9951887360,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.27.ffn_up.weight",
"offset": 10049372160,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 10244341760,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.27.attn_output.weight",
"offset": 10244362240,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.27.attn_qkv.weight",
"offset": 10272215040,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.28.attn_output.weight",
"offset": 10313994240,
"shape": [
5120,
5120
],
"size": 41779200,
"type": 8
},
{
"name": "blk.28.attn_qkv.weight",
"offset": 10341847040,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.28.attn_norm.weight",
"offset": 10383626240,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 10383646720,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.28.ffn_up.weight",
"offset": 10481131520,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 10676101120,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.29.attn_norm.weight",
"offset": 10676121600,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 10676142080,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.29.ffn_up.weight",
"offset": 10773626880,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 10968596480,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.29.attn_output.weight",
"offset": 10968616960,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.29.attn_qkv.weight",
"offset": 10996469760,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.30.attn_norm.weight",
"offset": 11038248960,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 11038269440,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.30.ffn_up.weight",
"offset": 11135754240,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 11330723840,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.30.attn_output.weight",
"offset": 11330744320,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.30.attn_qkv.weight",
"offset": 11358597120,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.31.attn_norm.weight",
"offset": 11400376320,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 11400396800,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.31.ffn_up.weight",
"offset": 11497881600,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 11692851200,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.31.attn_output.weight",
"offset": 11692871680,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.31.attn_qkv.weight",
"offset": 11720724480,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.32.attn_norm.weight",
"offset": 11762503680,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 11762524160,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.32.ffn_up.weight",
"offset": 11860008960,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 12054978560,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.32.attn_output.weight",
"offset": 12054999040,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.32.attn_qkv.weight",
"offset": 12082851840,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.33.attn_norm.weight",
"offset": 12124631040,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 12124651520,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.33.ffn_up.weight",
"offset": 12222136320,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 12417105920,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.33.attn_output.weight",
"offset": 12417126400,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.33.attn_qkv.weight",
"offset": 12444979200,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.34.attn_norm.weight",
"offset": 12486758400,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 12486778880,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.34.ffn_up.weight",
"offset": 12584263680,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 12779233280,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.34.attn_output.weight",
"offset": 12779253760,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.34.attn_qkv.weight",
"offset": 12807106560,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.35.attn_output.weight",
"offset": 12848885760,
"shape": [
5120,
5120
],
"size": 41779200,
"type": 8
},
{
"name": "blk.35.attn_qkv.weight",
"offset": 12876738560,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "output.weight",
"offset": 12918517760,
"shape": [
5120,
32064
],
"size": 41779200,
"type": 8
},
{
"name": "blk.35.attn_norm.weight",
"offset": 13092945920,
"shape": [
5120
],
"size": 174428160,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 13092966400,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.35.ffn_up.weight",
"offset": 13190451200,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 13385420800,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.36.attn_norm.weight",
"offset": 13385441280,
"shape": [
5120
],
"size": 20480,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 13385461760,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.36.ffn_up.weight",
"offset": 13482946560,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 13677916160,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.36.attn_output.weight",
"offset": 13677936640,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.36.attn_qkv.weight",
"offset": 13705789440,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.37.attn_norm.weight",
"offset": 13747568640,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 13747589120,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.37.ffn_up.weight",
"offset": 13845073920,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 14040043520,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.37.attn_output.weight",
"offset": 14040064000,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.37.attn_qkv.weight",
"offset": 14067916800,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.38.attn_norm.weight",
"offset": 14109696000,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 14109716480,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.38.ffn_up.weight",
"offset": 14207201280,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 14402170880,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.38.attn_output.weight",
"offset": 14402191360,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.38.attn_qkv.weight",
"offset": 14430044160,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "blk.39.attn_norm.weight",
"offset": 14471823360,
"shape": [
5120
],
"size": 41779200,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 14471843840,
"shape": [
17920,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.39.ffn_up.weight",
"offset": 14569328640,
"shape": [
5120,
35840
],
"size": 97484800,
"type": 8
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 14764298240,
"shape": [
5120
],
"size": 194969600,
"type": 0
},
{
"name": "blk.39.attn_output.weight",
"offset": 14764318720,
"shape": [
5120,
5120
],
"size": 20480,
"type": 8
},
{
"name": "blk.39.attn_qkv.weight",
"offset": 14792171520,
"shape": [
5120,
7680
],
"size": 27852800,
"type": 8
},
{
"name": "output_norm.weight",
"offset": 14833950720,
"shape": [
5120
],
"size": 41779200,
"type": 0
}
],
"version": 3
}