Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
ae8f58683991 · 5.3GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 15,
"general.name": "Yi-1.5-9B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 4,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 48,
"llama.context_length": 16384,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 11008,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 8829407232,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 147456000,
"shape": [
4096
],
"size": 147456000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 147472384,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 184459264,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.0.ffn_up.weight",
"offset": 209821696,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 235184128,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 235200512,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.0.attn_output.weight",
"offset": 236380160,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 245817344,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.0.attn_v.weight",
"offset": 255254528,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 256974848,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 256991232,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 293978112,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.1.ffn_up.weight",
"offset": 319340544,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 344702976,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 344719360,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.1.attn_output.weight",
"offset": 345899008,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 355336192,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.1.attn_v.weight",
"offset": 364773376,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 366493696,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 366510080,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 403496960,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.10.ffn_up.weight",
"offset": 428859392,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 454221824,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 454238208,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.10.attn_output.weight",
"offset": 455417856,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 464855040,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.10.attn_v.weight",
"offset": 474292224,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 476012544,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 476028928,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 513015808,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.11.ffn_up.weight",
"offset": 538378240,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 563740672,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 563757056,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.11.attn_output.weight",
"offset": 564936704,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 574373888,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.11.attn_v.weight",
"offset": 583811072,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 585531392,
"shape": [
4096,
11008
],
"size": 1720320,
"type": 12
},
{
"name": "blk.12.ffn_up.weight",
"offset": 610893824,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.12.attn_k.weight",
"offset": 636256256,
"shape": [
4096,
512
],
"size": 25362432,
"type": 12
},
{
"name": "blk.12.attn_output.weight",
"offset": 637435904,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 646873088,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.12.attn_v.weight",
"offset": 656310272,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 658030592,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 658046976,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 695033856,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.2.ffn_up.weight",
"offset": 720396288,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 745758720,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 745775104,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.2.attn_output.weight",
"offset": 746954752,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 756391936,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.2.attn_v.weight",
"offset": 765829120,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 767549440,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 767565824,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 804552704,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.3.ffn_up.weight",
"offset": 829915136,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 855277568,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 855293952,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.3.attn_output.weight",
"offset": 856473600,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 865910784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.3.attn_v.weight",
"offset": 875347968,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.4.attn_norm.weight",
"offset": 876527616,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 876544000,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 901906432,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.4.ffn_up.weight",
"offset": 927268864,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 952631296,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 952647680,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.attn_output.weight",
"offset": 953827328,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 963264512,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.4.attn_v.weight",
"offset": 972701696,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 973881344,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 973897728,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 999260160,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1024622592,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1049985024,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1050001408,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.attn_output.weight",
"offset": 1051181056,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 1060618240,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.5.attn_v.weight",
"offset": 1070055424,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1071775744,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1071792128,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1108779008,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1134141440,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1159503872,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1159520256,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.6.attn_output.weight",
"offset": 1160699904,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 1170137088,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.6.attn_v.weight",
"offset": 1179574272,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1180753920,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1180770304,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1206132736,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1231495168,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1256857600,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1256873984,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.attn_output.weight",
"offset": 1258053632,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 1267490816,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.7.attn_v.weight",
"offset": 1276928000,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1278107648,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1278124032,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1303486464,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1328848896,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 1354211328,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 1354227712,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.attn_output.weight",
"offset": 1355407360,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 1364844544,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.8.attn_v.weight",
"offset": 1374281728,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 1376002048,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 1376018432,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 1413005312,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.9.ffn_up.weight",
"offset": 1438367744,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 1463730176,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 1463746560,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.attn_output.weight",
"offset": 1464926208,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 1474363392,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.9.attn_v.weight",
"offset": 1483800576,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1484980224,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1484996608,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1510359040,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1510375424,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1510391808,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1535754240,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1561116672,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1586479104,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1586495488,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.attn_output.weight",
"offset": 1587675136,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 1597112320,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.13.attn_v.weight",
"offset": 1606549504,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1607729152,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1607745536,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1644732416,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1670094848,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1695457280,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1695473664,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.14.attn_output.weight",
"offset": 1696653312,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 1706090496,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.14.attn_v.weight",
"offset": 1715527680,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 1717248000,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 1717264384,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 1742626816,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.15.ffn_up.weight",
"offset": 1767989248,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 1793351680,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 1793368064,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.attn_output.weight",
"offset": 1794547712,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 1803984896,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.15.attn_v.weight",
"offset": 1813422080,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 1814601728,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 1814618112,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 1839980544,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 1865342976,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 1890705408,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 1890721792,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.attn_output.weight",
"offset": 1891901440,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 1901338624,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.16.attn_v.weight",
"offset": 1910775808,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.17.attn_norm.weight",
"offset": 1911955456,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 1911971840,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 1948958720,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.17.ffn_up.weight",
"offset": 1974321152,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 1999683584,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 1999699968,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.17.attn_output.weight",
"offset": 2000879616,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 2010316800,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.17.attn_v.weight",
"offset": 2019753984,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2021474304,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2021490688,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 2046853120,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.18.ffn_up.weight",
"offset": 2072215552,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 2097577984,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 2097594368,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.attn_output.weight",
"offset": 2098774016,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 2108211200,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.18.attn_v.weight",
"offset": 2117648384,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 2118828032,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 2118844416,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 2144206848,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.19.ffn_up.weight",
"offset": 2169569280,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 2194931712,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 2194948096,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.attn_output.weight",
"offset": 2196127744,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 2205564928,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.19.attn_v.weight",
"offset": 2215002112,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 2216181760,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 2216198144,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 2253185024,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.20.ffn_up.weight",
"offset": 2278547456,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 2303909888,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 2303926272,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.attn_output.weight",
"offset": 2305105920,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 2314543104,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.20.attn_v.weight",
"offset": 2323980288,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 2325700608,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 2325716992,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 2351079424,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.21.ffn_up.weight",
"offset": 2376441856,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 2401804288,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 2401820672,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.attn_output.weight",
"offset": 2403000320,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 2412437504,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.21.attn_v.weight",
"offset": 2421874688,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 2423054336,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 2423070720,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 2448433152,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.22.ffn_up.weight",
"offset": 2473795584,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 2499158016,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 2499174400,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.attn_output.weight",
"offset": 2500354048,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 2509791232,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.22.attn_v.weight",
"offset": 2519228416,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 2520408064,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 2520424448,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 2557411328,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.23.ffn_up.weight",
"offset": 2582773760,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 2608136192,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 2608152576,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.attn_output.weight",
"offset": 2609332224,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 2618769408,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.23.attn_v.weight",
"offset": 2628206592,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 2629926912,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 2629943296,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 2655305728,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.24.ffn_up.weight",
"offset": 2680668160,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 2706030592,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 2706046976,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.attn_output.weight",
"offset": 2707226624,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 2716663808,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.24.attn_v.weight",
"offset": 2726100992,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 2727280640,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 2727297024,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 2752659456,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.25.ffn_up.weight",
"offset": 2778021888,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 2803384320,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 2803400704,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.attn_output.weight",
"offset": 2804580352,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 2814017536,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.25.attn_v.weight",
"offset": 2823454720,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.26.attn_norm.weight",
"offset": 2824634368,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 2824650752,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 2861637632,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.26.ffn_up.weight",
"offset": 2887000064,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 2912362496,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 2912378880,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.attn_output.weight",
"offset": 2913558528,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 2922995712,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.26.attn_v.weight",
"offset": 2932432896,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.27.attn_k.weight",
"offset": 2934153216,
"shape": [
4096,
512
],
"size": 1720320,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 2935332864,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.27.attn_v.weight",
"offset": 2944770048,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 2945949696,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 2945966080,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 2971328512,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.27.ffn_up.weight",
"offset": 2996690944,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 3022053376,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.27.attn_output.weight",
"offset": 3022069760,
"shape": [
4096,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 3031506944,
"shape": [
4096
],
"size": 9437184,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 3031523328,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 3056885760,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.28.ffn_up.weight",
"offset": 3082248192,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 3107610624,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 3107627008,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.attn_output.weight",
"offset": 3108806656,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 3118243840,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.28.attn_v.weight",
"offset": 3127681024,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 3128860672,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 3128877056,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 3165863936,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 3191226368,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 3216588800,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 3216605184,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.29.attn_output.weight",
"offset": 3217784832,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 3227222016,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.29.attn_v.weight",
"offset": 3236659200,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 3238379520,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 3238395904,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 3263758336,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.30.ffn_up.weight",
"offset": 3289120768,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 3314483200,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 3314499584,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.30.attn_output.weight",
"offset": 3315679232,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 3325116416,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.30.attn_v.weight",
"offset": 3334553600,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.31.attn_norm.weight",
"offset": 3335733248,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 3335749632,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 3361112064,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.31.ffn_up.weight",
"offset": 3386474496,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 3411836928,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 3411853312,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.31.attn_output.weight",
"offset": 3413032960,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 3422470144,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.31.attn_v.weight",
"offset": 3431907328,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.32.attn_norm.weight",
"offset": 3433086976,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 3433103360,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 3470090240,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.32.ffn_up.weight",
"offset": 3495452672,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "\u0001\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000`\ufffd\ufffd\u0000\u0000",
"offset": 3520815104,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 3520831488,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.32.attn_output.weight",
"offset": 3522011136,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.32.attn_q.weight",
"offset": 3531448320,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.32.attn_v.weight",
"offset": 3540885504,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 3542605824,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 3542622208,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 3567984640,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.33.ffn_up.weight",
"offset": 3593347072,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 3618709504,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 3618725888,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.33.attn_output.weight",
"offset": 3619905536,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.33.attn_q.weight",
"offset": 3629342720,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.33.attn_v.weight",
"offset": 3638779904,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.34.attn_norm.weight",
"offset": 3639959552,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 3639975936,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 3665338368,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.34.ffn_up.weight",
"offset": 3690700800,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 3716063232,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 3716079616,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.34.attn_output.weight",
"offset": 3717259264,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.34.attn_q.weight",
"offset": 3726696448,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.34.attn_v.weight",
"offset": 3736133632,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.35.attn_norm.weight",
"offset": 3737313280,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 3737329664,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 3774316544,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.35.ffn_up.weight",
"offset": 3799678976,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 3825041408,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 3825057792,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.35.attn_output.weight",
"offset": 3826237440,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.35.attn_q.weight",
"offset": 3835674624,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.35.attn_v.weight",
"offset": 3845111808,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.36.attn_norm.weight",
"offset": 3846832128,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 3846848512,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 3872210944,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.36.ffn_up.weight",
"offset": 3897573376,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 3922935808,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 3922952192,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.36.attn_output.weight",
"offset": 3924131840,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.36.attn_q.weight",
"offset": 3933569024,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.36.attn_v.weight",
"offset": 3943006208,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.37.attn_norm.weight",
"offset": 3944185856,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 3944202240,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 3969564672,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.37.ffn_up.weight",
"offset": 3994927104,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 4020289536,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 4020305920,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.37.attn_output.weight",
"offset": 4021485568,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.37.attn_q.weight",
"offset": 4030922752,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.37.attn_v.weight",
"offset": 4040359936,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.38.attn_norm.weight",
"offset": 4041539584,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 4041555968,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 4078542848,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.38.ffn_up.weight",
"offset": 4103905280,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 4129267712,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 4129284096,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.38.attn_output.weight",
"offset": 4130463744,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.38.attn_q.weight",
"offset": 4139900928,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.38.attn_v.weight",
"offset": 4149338112,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.39.attn_norm.weight",
"offset": 4151058432,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 4151074816,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 4176437248,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.39.ffn_up.weight",
"offset": 4201799680,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 4227162112,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 4227178496,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.39.attn_output.weight",
"offset": 4228358144,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.39.attn_q.weight",
"offset": 4237795328,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.39.attn_v.weight",
"offset": 4247232512,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.40.attn_norm.weight",
"offset": 4248412160,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 4248428544,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 4273790976,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.40.ffn_up.weight",
"offset": 4299153408,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 4324515840,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 4324532224,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.40.attn_output.weight",
"offset": 4325711872,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.40.attn_q.weight",
"offset": 4335149056,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.40.attn_v.weight",
"offset": 4344586240,
"shape": [
4096,
512
],
"size": 9437184,
"type": 12
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 4345765888,
"shape": [
4096,
11008
],
"size": 1179648,
"type": 12
},
{
"name": "blk.41.attn_k.weight",
"offset": 4371128320,
"shape": [
4096,
512
],
"size": 25362432,
"type": 12
},
{
"name": "blk.41.attn_output.weight",
"offset": 4372307968,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.41.attn_q.weight",
"offset": 4381745152,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.41.attn_v.weight",
"offset": 4391182336,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "output.weight",
"offset": 4392902656,
"shape": [
4096,
64000
],
"size": 1720320,
"type": 14
},
{
"name": "blk.41.attn_norm.weight",
"offset": 4607942656,
"shape": [
4096
],
"size": 215040000,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 4607959040,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.41.ffn_up.weight",
"offset": 4644945920,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 4670308352,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.42.attn_norm.weight",
"offset": 4670324736,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 4670341120,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 4707328000,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.42.ffn_up.weight",
"offset": 4732690432,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 4758052864,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 4758069248,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.42.attn_output.weight",
"offset": 4759248896,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.42.attn_q.weight",
"offset": 4768686080,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.42.attn_v.weight",
"offset": 4778123264,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.43.attn_norm.weight",
"offset": 4779843584,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 4779859968,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 4816846848,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.43.ffn_up.weight",
"offset": 4842209280,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 4867571712,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 4867588096,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.43.attn_output.weight",
"offset": 4868767744,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.43.attn_q.weight",
"offset": 4878204928,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.43.attn_v.weight",
"offset": 4887642112,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.44.attn_norm.weight",
"offset": 4889362432,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 4889378816,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 4926365696,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.44.ffn_up.weight",
"offset": 4951728128,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 4977090560,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 4977106944,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.44.attn_output.weight",
"offset": 4978286592,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.44.attn_q.weight",
"offset": 4987723776,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.44.attn_v.weight",
"offset": 4997160960,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 4998881280,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 4998897664,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 5035884544,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.45.ffn_up.weight",
"offset": 5061246976,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 5086609408,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 5086625792,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.45.attn_output.weight",
"offset": 5087805440,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.45.attn_q.weight",
"offset": 5097242624,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.45.attn_v.weight",
"offset": 5106679808,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.46.attn_norm.weight",
"offset": 5108400128,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 5108416512,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 5145403392,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.46.ffn_up.weight",
"offset": 5170765824,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 5196128256,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 5196144640,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.46.attn_output.weight",
"offset": 5197324288,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.46.attn_q.weight",
"offset": 5206761472,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.46.attn_v.weight",
"offset": 5216198656,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "blk.47.attn_norm.weight",
"offset": 5217918976,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 5217935360,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 5254922240,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 12
},
{
"name": "blk.47.ffn_up.weight",
"offset": 5280284672,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 12
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 5305647104,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "\u0007~ww\ufffd\u0000\u0007\ufffd\u0000\ufffdw~\u0000pwppw\u0007w",
"offset": 5305663488,
"shape": [
4096,
512
],
"size": 16384,
"type": 12
},
{
"name": "blk.47.attn_output.weight",
"offset": 5306843136,
"shape": [
4096,
4096
],
"size": 1179648,
"type": 12
},
{
"name": "blk.47.attn_q.weight",
"offset": 5316280320,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 12
},
{
"name": "blk.47.attn_v.weight",
"offset": 5325717504,
"shape": [
4096,
512
],
"size": 9437184,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 5327437824,
"shape": [
4096
],
"size": 1720320,
"type": 0
}
],
"version": 3
}