Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
6d83b3ba97f1 · 6.3GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 17,
"general.name": "Yi-1.5-9B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 4,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 48,
"llama.context_length": 16384,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 11008,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 8829407232,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 180224000,
"shape": [
4096
],
"size": 180224000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 180240384,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 217227264,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.0.ffn_up.weight",
"offset": 248225792,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 279224320,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 279240704,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 280682496,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.0.attn_q.weight",
"offset": 292216832,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.0.attn_v.weight",
"offset": 303751168,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 305471488,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 305487872,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 342474752,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.1.ffn_up.weight",
"offset": 373473280,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 404471808,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 404488192,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 405929984,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.1.attn_q.weight",
"offset": 417464320,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.1.attn_v.weight",
"offset": 428998656,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 430718976,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 430735360,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 467722240,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.10.ffn_up.weight",
"offset": 498720768,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 529719296,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 529735680,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 531177472,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.10.attn_q.weight",
"offset": 542711808,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.10.attn_v.weight",
"offset": 554246144,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 555966464,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 555982848,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 592969728,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.11.ffn_up.weight",
"offset": 623968256,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 654966784,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 654983168,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 656424960,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.11.attn_q.weight",
"offset": 667959296,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.11.attn_v.weight",
"offset": 679493632,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 681213952,
"shape": [
4096,
11008
],
"size": 1720320,
"type": 13
},
{
"name": "blk.12.ffn_up.weight",
"offset": 712212480,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.12.attn_k.weight",
"offset": 743211008,
"shape": [
4096,
512
],
"size": 30998528,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 744652800,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.12.attn_q.weight",
"offset": 756187136,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_v.weight",
"offset": 767721472,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 769441792,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 769458176,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 806445056,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.2.ffn_up.weight",
"offset": 837443584,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 868442112,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 868458496,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 869900288,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.2.attn_q.weight",
"offset": 881434624,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.2.attn_v.weight",
"offset": 892968960,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 894689280,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 894705664,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 931692544,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.3.ffn_up.weight",
"offset": 962691072,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 993689600,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 993705984,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 995147776,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.3.attn_q.weight",
"offset": 1006682112,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.3.attn_v.weight",
"offset": 1018216448,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1019658240,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1019674624,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1050673152,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1081671680,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1112670208,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1112686592,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 1114128384,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.4.attn_q.weight",
"offset": 1125662720,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.4.attn_v.weight",
"offset": 1137197056,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1138638848,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1138655232,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1169653760,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1200652288,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1231650816,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1231667200,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 1233108992,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.5.attn_q.weight",
"offset": 1244643328,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.5.attn_v.weight",
"offset": 1256177664,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1257897984,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1257914368,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1294901248,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1325899776,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1356898304,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1356914688,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 1358356480,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "\u0000\u0000\r\u0000\u0000\u0000\u0000\u0000\u000f\ufffd\u0000\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000",
"offset": 1369890816,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.6.attn_v.weight",
"offset": 1381425152,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.7.attn_norm.weight",
"offset": 1382866944,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 1382883328,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1413881856,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1444880384,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 1475878912,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 1475895296,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 1477337088,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.7.attn_q.weight",
"offset": 1488871424,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.7.attn_v.weight",
"offset": 1500405760,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1501847552,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1501863936,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1532862464,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1563860992,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 1594859520,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 1594875904,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 1596317696,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.8.attn_q.weight",
"offset": 1607852032,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.8.attn_v.weight",
"offset": 1619386368,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 1621106688,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 1621123072,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 1658109952,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.9.ffn_up.weight",
"offset": 1689108480,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 1720107008,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 1720123392,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 1721565184,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.9.attn_q.weight",
"offset": 1733099520,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.9.attn_v.weight",
"offset": 1744633856,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1746075648,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1746092032,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1777090560,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1777106944,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1777123328,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1808121856,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1839120384,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1870118912,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1870135296,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 1871577088,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.13.attn_q.weight",
"offset": 1883111424,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.13.attn_v.weight",
"offset": 1894645760,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1896087552,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1896103936,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1933090816,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1964089344,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1995087872,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1995104256,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 1996546048,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.14.attn_q.weight",
"offset": 2008080384,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.14.attn_v.weight",
"offset": 2019614720,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 2021335040,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 2021351424,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 2052349952,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.15.ffn_up.weight",
"offset": 2083348480,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 2114347008,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 2114363392,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 2115805184,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.15.attn_q.weight",
"offset": 2127339520,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.15.attn_v.weight",
"offset": 2138873856,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_norm.weight",
"offset": 2140315648,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 2140332032,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 2171330560,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.16.ffn_up.weight",
"offset": 2202329088,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 2233327616,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 2233344000,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 2234785792,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.16.attn_q.weight",
"offset": 2246320128,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.16.attn_v.weight",
"offset": 2257854464,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_norm.weight",
"offset": 2259296256,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 2259312640,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 2296299520,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.17.ffn_up.weight",
"offset": 2327298048,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 2358296576,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 2358312960,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 2359754752,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.17.attn_q.weight",
"offset": 2371289088,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.17.attn_v.weight",
"offset": 2382823424,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 2384543744,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 2384560128,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 2415558656,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.18.ffn_up.weight",
"offset": 2446557184,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 2477555712,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 2477572096,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 2479013888,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.18.attn_q.weight",
"offset": 2490548224,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.18.attn_v.weight",
"offset": 2502082560,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_norm.weight",
"offset": 2503524352,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 2503540736,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 2534539264,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.19.ffn_up.weight",
"offset": 2565537792,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 2596536320,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 2596552704,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 2597994496,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.19.attn_q.weight",
"offset": 2609528832,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.19.attn_v.weight",
"offset": 2621063168,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_norm.weight",
"offset": 2622504960,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 2622521344,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 2659508224,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.20.ffn_up.weight",
"offset": 2690506752,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 2721505280,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 2721521664,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 2722963456,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.20.attn_q.weight",
"offset": 2734497792,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.20.attn_v.weight",
"offset": 2746032128,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 2747752448,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 2747768832,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 2778767360,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.21.ffn_up.weight",
"offset": 2809765888,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 2840764416,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 2840780800,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 2842222592,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.21.attn_q.weight",
"offset": 2853756928,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.21.attn_v.weight",
"offset": 2865291264,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.22.attn_norm.weight",
"offset": 2866733056,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 2866749440,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 2897747968,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.22.ffn_up.weight",
"offset": 2928746496,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 2959745024,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 2959761408,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 2961203200,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.22.attn_q.weight",
"offset": 2972737536,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.22.attn_v.weight",
"offset": 2984271872,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_norm.weight",
"offset": 2985713664,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 2985730048,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 3022716928,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.23.ffn_up.weight",
"offset": 3053715456,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 3084713984,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 3084730368,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 3086172160,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.23.attn_q.weight",
"offset": 3097706496,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.23.attn_v.weight",
"offset": 3109240832,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 3110961152,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 3110977536,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 3141976064,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.24.ffn_up.weight",
"offset": 3172974592,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 3203973120,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 3203989504,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 3205431296,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.24.attn_q.weight",
"offset": 3216965632,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.24.attn_v.weight",
"offset": 3228499968,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.25.attn_norm.weight",
"offset": 3229941760,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 3229958144,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 3260956672,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.25.ffn_up.weight",
"offset": 3291955200,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 3322953728,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 3322970112,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 3324411904,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.25.attn_q.weight",
"offset": 3335946240,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.25.attn_v.weight",
"offset": 3347480576,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_norm.weight",
"offset": 3348922368,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 3348938752,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 3385925632,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.26.ffn_up.weight",
"offset": 3416924160,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 3447922688,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 3447939072,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 3449380864,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.26.attn_q.weight",
"offset": 3460915200,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.26.attn_v.weight",
"offset": 3472449536,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.27.attn_k.weight",
"offset": 3474169856,
"shape": [
4096,
512
],
"size": 1720320,
"type": 13
},
{
"name": "blk.27.attn_q.weight",
"offset": 3475611648,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.27.attn_v.weight",
"offset": 3487145984,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 3488587776,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 3488604160,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 3519602688,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.27.ffn_up.weight",
"offset": 3550601216,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 3581599744,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.27.attn_output.weight",
"offset": 3581616128,
"shape": [
4096,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.28.attn_norm.weight",
"offset": 3593150464,
"shape": [
4096
],
"size": 11534336,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 3593166848,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 3624165376,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.28.ffn_up.weight",
"offset": 3655163904,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 3686162432,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 3686178816,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 3687620608,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.28.attn_q.weight",
"offset": 3699154944,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.28.attn_v.weight",
"offset": 3710689280,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.29.attn_norm.weight",
"offset": 3712131072,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 3712147456,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 3749134336,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.29.ffn_up.weight",
"offset": 3780132864,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 3811131392,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 3811147776,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 3812589568,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.29.attn_q.weight",
"offset": 3824123904,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.29.attn_v.weight",
"offset": 3835658240,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 3837378560,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 3837394944,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 3868393472,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.30.ffn_up.weight",
"offset": 3899392000,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 3930390528,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 3930406912,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 3931848704,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.30.attn_q.weight",
"offset": 3943383040,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.30.attn_v.weight",
"offset": 3954917376,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.31.attn_norm.weight",
"offset": 3956359168,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 3956375552,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 3987374080,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.31.ffn_up.weight",
"offset": 4018372608,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 4049371136,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 4049387520,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 4050829312,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.31.attn_q.weight",
"offset": 4062363648,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.31.attn_v.weight",
"offset": 4073897984,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.32.attn_norm.weight",
"offset": 4075339776,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 4075356160,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 4112343040,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.32.ffn_up.weight",
"offset": 4143341568,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 4174340096,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 4174356480,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.32.attn_output.weight",
"offset": 4175798272,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.32.attn_q.weight",
"offset": 4187332608,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.32.attn_v.weight",
"offset": 4198866944,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 4200587264,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 4200603648,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 4231602176,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.33.ffn_up.weight",
"offset": 4262600704,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 4293599232,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 4293615616,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.33.attn_output.weight",
"offset": 4295057408,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.33.attn_q.weight",
"offset": 4306591744,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.33.attn_v.weight",
"offset": 4318126080,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.34.attn_norm.weight",
"offset": 4319567872,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 4319584256,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 4350582784,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.34.ffn_up.weight",
"offset": 4381581312,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 4412579840,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 4412596224,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.34.attn_output.weight",
"offset": 4414038016,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.34.attn_q.weight",
"offset": 4425572352,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.34.attn_v.weight",
"offset": 4437106688,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.35.attn_norm.weight",
"offset": 4438548480,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 4438564864,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 4475551744,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.35.ffn_up.weight",
"offset": 4506550272,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 4537548800,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 4537565184,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.35.attn_output.weight",
"offset": 4539006976,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.35.attn_q.weight",
"offset": 4550541312,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.35.attn_v.weight",
"offset": 4562075648,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.36.attn_norm.weight",
"offset": 4563795968,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 4563812352,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 4594810880,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.36.ffn_up.weight",
"offset": 4625809408,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 4656807936,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 4656824320,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.36.attn_output.weight",
"offset": 4658266112,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.36.attn_q.weight",
"offset": 4669800448,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.36.attn_v.weight",
"offset": 4681334784,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.37.attn_norm.weight",
"offset": 4682776576,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 4682792960,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 4713791488,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.37.ffn_up.weight",
"offset": 4744790016,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 4775788544,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 4775804928,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.37.attn_output.weight",
"offset": 4777246720,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.37.attn_q.weight",
"offset": 4788781056,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.37.attn_v.weight",
"offset": 4800315392,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.38.attn_norm.weight",
"offset": 4801757184,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 4801773568,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 4838760448,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.38.ffn_up.weight",
"offset": 4869758976,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 4900757504,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 4900773888,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.38.attn_output.weight",
"offset": 4902215680,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.38.attn_q.weight",
"offset": 4913750016,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.38.attn_v.weight",
"offset": 4925284352,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.39.attn_norm.weight",
"offset": 4927004672,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 4927021056,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 4958019584,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.39.ffn_up.weight",
"offset": 4989018112,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 5020016640,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 5020033024,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.39.attn_output.weight",
"offset": 5021474816,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.39.attn_q.weight",
"offset": 5033009152,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.39.attn_v.weight",
"offset": 5044543488,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.40.attn_norm.weight",
"offset": 5045985280,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 5046001664,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 5077000192,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.40.ffn_up.weight",
"offset": 5107998720,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 5138997248,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 5139013632,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.40.attn_output.weight",
"offset": 5140455424,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.40.attn_q.weight",
"offset": 5151989760,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.40.attn_v.weight",
"offset": 5163524096,
"shape": [
4096,
512
],
"size": 11534336,
"type": 13
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 5164965888,
"shape": [
4096,
11008
],
"size": 1441792,
"type": 13
},
{
"name": "blk.41.attn_k.weight",
"offset": 5195964416,
"shape": [
4096,
512
],
"size": 30998528,
"type": 13
},
{
"name": "blk.41.attn_output.weight",
"offset": 5197406208,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.41.attn_q.weight",
"offset": 5208940544,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.41.attn_v.weight",
"offset": 5220474880,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "output.weight",
"offset": 5222195200,
"shape": [
4096,
64000
],
"size": 1720320,
"type": 14
},
{
"name": "blk.41.attn_norm.weight",
"offset": 5437235200,
"shape": [
4096
],
"size": 215040000,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 5437251584,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.41.ffn_up.weight",
"offset": 5474238464,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 5505236992,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.42.attn_norm.weight",
"offset": 5505253376,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 5505269760,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 5542256640,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.42.ffn_up.weight",
"offset": 5573255168,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 5604253696,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 5604270080,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.42.attn_output.weight",
"offset": 5605711872,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.42.attn_q.weight",
"offset": 5617246208,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.42.attn_v.weight",
"offset": 5628780544,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.43.attn_norm.weight",
"offset": 5630500864,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 5630517248,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 5667504128,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.43.ffn_up.weight",
"offset": 5698502656,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 5729501184,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 5729517568,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.43.attn_output.weight",
"offset": 5730959360,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.43.attn_q.weight",
"offset": 5742493696,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.43.attn_v.weight",
"offset": 5754028032,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.44.attn_norm.weight",
"offset": 5755748352,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 5755764736,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 5792751616,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.44.ffn_up.weight",
"offset": 5823750144,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 5854748672,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 5854765056,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.44.attn_output.weight",
"offset": 5856206848,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.44.attn_q.weight",
"offset": 5867741184,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.44.attn_v.weight",
"offset": 5879275520,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 5880995840,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 5881012224,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 5917999104,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.45.ffn_up.weight",
"offset": 5948997632,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 5979996160,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 5980012544,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.45.attn_output.weight",
"offset": 5981454336,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.45.attn_q.weight",
"offset": 5992988672,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.45.attn_v.weight",
"offset": 6004523008,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.46.attn_norm.weight",
"offset": 6006243328,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 6006259712,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 6043246592,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.46.ffn_up.weight",
"offset": 6074245120,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 6105243648,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 6105260032,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.46.attn_output.weight",
"offset": 6106701824,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.46.attn_q.weight",
"offset": 6118236160,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.46.attn_v.weight",
"offset": 6129770496,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "blk.47.attn_norm.weight",
"offset": 6131490816,
"shape": [
4096
],
"size": 1720320,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 6131507200,
"shape": [
11008,
4096
],
"size": 16384,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 6168494080,
"shape": [
4096,
11008
],
"size": 36986880,
"type": 13
},
{
"name": "blk.47.ffn_up.weight",
"offset": 6199492608,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 13
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 6230491136,
"shape": [
4096
],
"size": 30998528,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 6230507520,
"shape": [
4096,
512
],
"size": 16384,
"type": 13
},
{
"name": "blk.47.attn_output.weight",
"offset": 6231949312,
"shape": [
4096,
4096
],
"size": 1441792,
"type": 13
},
{
"name": "blk.47.attn_q.weight",
"offset": 6243483648,
"shape": [
4096,
4096
],
"size": 11534336,
"type": 13
},
{
"name": "blk.47.attn_v.weight",
"offset": 6255017984,
"shape": [
4096,
512
],
"size": 11534336,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 6256738304,
"shape": [
4096
],
"size": 1720320,
"type": 0
}
],
"version": 3
}