Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
dee4c06fce8a · 4.3GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 12,
"general.name": "Yi-1.5-9B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 32,
"llama.attention.head_count_kv": 4,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 48,
"llama.context_length": 16384,
"llama.embedding_length": 4096,
"llama.feed_forward_length": 11008,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 8829407232,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 112640000,
"shape": [
4096
],
"size": 112640000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 112656384,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 143654912,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 11
},
{
"name": "blk.0.ffn_up.weight",
"offset": 163028992,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 182403072,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 182419456,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.0.attn_output.weight",
"offset": 183320576,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 192757760,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.0.attn_v.weight",
"offset": 199966720,
"shape": [
4096,
512
],
"size": 7208960,
"type": 13
},
{
"name": "blk.1.attn_norm.weight",
"offset": 201408512,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 201424896,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 232423424,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 11
},
{
"name": "blk.1.ffn_up.weight",
"offset": 251797504,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 271171584,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 271187968,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "attn_norm.weight\u0001\u0000\u0000\u0000\u0000\u0010\u0000\u0000",
"offset": 272089088,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 281526272,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.1.attn_v.weight",
"offset": 288735232,
"shape": [
4096,
512
],
"size": 7208960,
"type": 13
},
{
"name": "blk.10.attn_norm.weight",
"offset": 290177024,
"shape": [
4096
],
"size": 1441792,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 290193408,
"shape": [
11008,
4096
],
"size": 16384,
"type": 13
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 321191936,
"shape": [
4096,
11008
],
"size": 30998528,
"type": 11
},
{
"name": "blk.10.ffn_up.weight",
"offset": 340566016,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 359940096,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 359956480,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.10.attn_output.weight",
"offset": 360857600,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 370294784,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.10.attn_v.weight",
"offset": 377503744,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 378683392,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 378699776,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 404062208,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.11.ffn_up.weight",
"offset": 423436288,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 442810368,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 442826752,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.11.attn_output.weight",
"offset": 443727872,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 453165056,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.11.attn_v.weight",
"offset": 460374016,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 461553664,
"shape": [
4096,
11008
],
"size": 1179648,
"type": 11
},
{
"name": "blk.12.ffn_up.weight",
"offset": 480927744,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.12.attn_k.weight",
"offset": 500301824,
"shape": [
4096,
512
],
"size": 19374080,
"type": 11
},
{
"name": "blk.12.attn_output.weight",
"offset": 501202944,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 510640128,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.12.attn_v.weight",
"offset": 517849088,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.2.attn_norm.weight",
"offset": 519028736,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 519045120,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 544407552,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.2.ffn_up.weight",
"offset": 563781632,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 583155712,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 583172096,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.2.attn_output.weight",
"offset": 584073216,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 593510400,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.2.attn_v.weight",
"offset": 600719360,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.3.attn_norm.weight",
"offset": 601899008,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 601915392,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 627277824,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.3.ffn_up.weight",
"offset": 646651904,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 666025984,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 666042368,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.3.attn_output.weight",
"offset": 666943488,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 676380672,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.3.attn_v.weight",
"offset": 683589632,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.4.attn_norm.weight",
"offset": 684769280,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 684785664,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 710148096,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.4.ffn_up.weight",
"offset": 729522176,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 748896256,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 748912640,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.4.attn_output.weight",
"offset": 749813760,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 759250944,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.4.attn_v.weight",
"offset": 766459904,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 767639552,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 767655936,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 793018368,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.5.ffn_up.weight",
"offset": 812392448,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 831766528,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 831782912,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.5.attn_output.weight",
"offset": 832684032,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 842121216,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.5.attn_v.weight",
"offset": 849330176,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 850509824,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 850526208,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 875888640,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.6.ffn_up.weight",
"offset": 895262720,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 914636800,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 914653184,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.6.attn_output.weight",
"offset": 915554304,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 924991488,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.6.attn_v.weight",
"offset": 932200448,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 933380096,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 933396480,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 958758912,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.7.ffn_up.weight",
"offset": 978132992,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 997507072,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 997523456,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.7.attn_output.weight",
"offset": 998424576,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 1007861760,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.7.attn_v.weight",
"offset": 1015070720,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.8.attn_norm.weight",
"offset": 1016250368,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 1016266752,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 1041629184,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.8.ffn_up.weight",
"offset": 1061003264,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 1080377344,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 1080393728,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.8.attn_output.weight",
"offset": 1081294848,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 1090732032,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.8.attn_v.weight",
"offset": 1097940992,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 1099120640,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 1099137024,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 1124499456,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.9.ffn_up.weight",
"offset": 1143873536,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 1163247616,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 1163264000,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.9.attn_output.weight",
"offset": 1164165120,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 1173602304,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.9.attn_v.weight",
"offset": 1180811264,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 1181990912,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 1182007296,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 1207369728,
"shape": [
4096
],
"size": 25362432,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 1207386112,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 1207402496,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 1232764928,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.13.ffn_up.weight",
"offset": 1252139008,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 1271513088,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 1271529472,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.13.attn_output.weight",
"offset": 1272430592,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 1281867776,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.13.attn_v.weight",
"offset": 1289076736,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 1290256384,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 1290272768,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 1315635200,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.14.ffn_up.weight",
"offset": 1335009280,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 1354383360,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 1354399744,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.14.attn_output.weight",
"offset": 1355300864,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 1364738048,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.14.attn_v.weight",
"offset": 1371947008,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 1373126656,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 1373143040,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 1398505472,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.15.ffn_up.weight",
"offset": 1417879552,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 1437253632,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 1437270016,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.15.attn_output.weight",
"offset": 1438171136,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 1447608320,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.15.attn_v.weight",
"offset": 1454817280,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 1455996928,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 1456013312,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 1481375744,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.16.ffn_up.weight",
"offset": 1500749824,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 1520123904,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 1520140288,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.16.attn_output.weight",
"offset": 1521041408,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 1530478592,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.16.attn_v.weight",
"offset": 1537687552,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.17.attn_norm.weight",
"offset": 1538867200,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 1538883584,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "\u0000\u0000\u0000+\u0000\u0000\u0000\u0000\u0000\u0000\u000b\u0000\u0000\u0000\u0000 d^\u0000\u0000\u0000\u0000",
"offset": 1564246016,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.17.ffn_up.weight",
"offset": 1583620096,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 1602994176,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 1603010560,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.17.attn_output.weight",
"offset": 1603911680,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 1613348864,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.17.attn_v.weight",
"offset": 1620557824,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 1621737472,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 1621753856,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 1647116288,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.18.ffn_up.weight",
"offset": 1666490368,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 1685864448,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 1685880832,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.18.attn_output.weight",
"offset": 1686781952,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 1696219136,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.18.attn_v.weight",
"offset": 1703428096,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 1704607744,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 1704624128,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 1729986560,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.19.ffn_up.weight",
"offset": 1749360640,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 1768734720,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 1768751104,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.19.attn_output.weight",
"offset": 1769652224,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 1779089408,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.19.attn_v.weight",
"offset": 1786298368,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 1787478016,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 1787494400,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 1812856832,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.20.ffn_up.weight",
"offset": 1832230912,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 1851604992,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 1851621376,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.20.attn_output.weight",
"offset": 1852522496,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 1861959680,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.20.attn_v.weight",
"offset": 1869168640,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.21.attn_norm.weight",
"offset": 1870348288,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 1870364672,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 1895727104,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.21.ffn_up.weight",
"offset": 1915101184,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 1934475264,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 1934491648,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.21.attn_output.weight",
"offset": 1935392768,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 1944829952,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.21.attn_v.weight",
"offset": 1952038912,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 1953218560,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 1953234944,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 1978597376,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.22.ffn_up.weight",
"offset": 1997971456,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 2017345536,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 2017361920,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.22.attn_output.weight",
"offset": 2018263040,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 2027700224,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.22.attn_v.weight",
"offset": 2034909184,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 2036088832,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 2036105216,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 2061467648,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.23.ffn_up.weight",
"offset": 2080841728,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 2100215808,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 2100232192,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.23.attn_output.weight",
"offset": 2101133312,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 2110570496,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.23.attn_v.weight",
"offset": 2117779456,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 2118959104,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 2118975488,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 2144337920,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.24.ffn_up.weight",
"offset": 2163712000,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 2183086080,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 2183102464,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.24.attn_output.weight",
"offset": 2184003584,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 2193440768,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.24.attn_v.weight",
"offset": 2200649728,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 2201829376,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 2201845760,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 2227208192,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.25.ffn_up.weight",
"offset": 2246582272,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 2265956352,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 2265972736,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.25.attn_output.weight",
"offset": 2266873856,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 2276311040,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.25.attn_v.weight",
"offset": 2283520000,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.26.attn_norm.weight",
"offset": 2284699648,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 2284716032,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 2310078464,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.26.ffn_up.weight",
"offset": 2329452544,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 2348826624,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 2348843008,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.26.attn_output.weight",
"offset": 2349744128,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 2359181312,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.26.attn_v.weight",
"offset": 2366390272,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.27.attn_k.weight",
"offset": 2367569920,
"shape": [
4096,
512
],
"size": 1179648,
"type": 11
},
{
"name": "blk.27.attn_q.weight",
"offset": 2368471040,
"shape": [
4096,
4096
],
"size": 901120,
"type": 11
},
{
"name": "blk.27.attn_v.weight",
"offset": 2375680000,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 2376859648,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 2376876032,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 2402238464,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.27.ffn_up.weight",
"offset": 2421612544,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 2440986624,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.27.attn_output.weight",
"offset": 2441003008,
"shape": [
4096,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 2450440192,
"shape": [
4096
],
"size": 9437184,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 2450456576,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 2475819008,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.28.ffn_up.weight",
"offset": 2495193088,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 2514567168,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 2514583552,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.28.attn_output.weight",
"offset": 2515484672,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 2524921856,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.28.attn_v.weight",
"offset": 2532130816,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 2533310464,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 2533326848,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 2558689280,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.29.ffn_up.weight",
"offset": 2578063360,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 2597437440,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 2597453824,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.29.attn_output.weight",
"offset": 2598354944,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 2607792128,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.29.attn_v.weight",
"offset": 2615001088,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.30.attn_norm.weight",
"offset": 2616180736,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 2616197120,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 2641559552,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.30.ffn_up.weight",
"offset": 2660933632,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 2680307712,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 2680324096,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.30.attn_output.weight",
"offset": 2681225216,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 2690662400,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.30.attn_v.weight",
"offset": 2697871360,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.31.attn_norm.weight",
"offset": 2699051008,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 2699067392,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 2724429824,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.31.ffn_up.weight",
"offset": 2743803904,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 2763177984,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 2763194368,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.31.attn_output.weight",
"offset": 2764095488,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 2773532672,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.31.attn_v.weight",
"offset": 2780741632,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.32.attn_norm.weight",
"offset": 2781921280,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 2781937664,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 2807300096,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.32.ffn_up.weight",
"offset": 2826674176,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "eight\u0002\u0000\u0000\u0000\u0000\u0010\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002\u0000\u0000\u0000",
"offset": 2846048256,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 2846064640,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.32.attn_output.weight",
"offset": 2846965760,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.32.attn_q.weight",
"offset": 2856402944,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.32.attn_v.weight",
"offset": 2863611904,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.33.attn_norm.weight",
"offset": 2864791552,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 2864807936,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 2890170368,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.33.ffn_up.weight",
"offset": 2909544448,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 2928918528,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 2928934912,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.33.attn_output.weight",
"offset": 2929836032,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.33.attn_q.weight",
"offset": 2939273216,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.33.attn_v.weight",
"offset": 2946482176,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.34.attn_norm.weight",
"offset": 2947661824,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 2947678208,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 2973040640,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.34.ffn_up.weight",
"offset": 2992414720,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 3011788800,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 3011805184,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.34.attn_output.weight",
"offset": 3012706304,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.34.attn_q.weight",
"offset": 3022143488,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.34.attn_v.weight",
"offset": 3029352448,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.35.attn_norm.weight",
"offset": 3030532096,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 3030548480,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 3055910912,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.35.ffn_up.weight",
"offset": 3075284992,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 3094659072,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 3094675456,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.35.attn_output.weight",
"offset": 3095576576,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.35.attn_q.weight",
"offset": 3105013760,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.35.attn_v.weight",
"offset": 3112222720,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.36.attn_norm.weight",
"offset": 3113402368,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 3113418752,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 3138781184,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.36.ffn_up.weight",
"offset": 3158155264,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 3177529344,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 3177545728,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.36.attn_output.weight",
"offset": 3178446848,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.36.attn_q.weight",
"offset": 3187884032,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.36.attn_v.weight",
"offset": 3195092992,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.37.attn_norm.weight",
"offset": 3196272640,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 3196289024,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 3221651456,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.37.ffn_up.weight",
"offset": 3241025536,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 3260399616,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 3260416000,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.37.attn_output.weight",
"offset": 3261317120,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.37.attn_q.weight",
"offset": 3270754304,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.37.attn_v.weight",
"offset": 3277963264,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.38.attn_norm.weight",
"offset": 3279142912,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 3279159296,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 3304521728,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.38.ffn_up.weight",
"offset": 3323895808,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 3343269888,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 3343286272,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.38.attn_output.weight",
"offset": 3344187392,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.38.attn_q.weight",
"offset": 3353624576,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.38.attn_v.weight",
"offset": 3360833536,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.39.attn_norm.weight",
"offset": 3362013184,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 3362029568,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 3387392000,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.39.ffn_up.weight",
"offset": 3406766080,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 3426140160,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 3426156544,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.39.attn_output.weight",
"offset": 3427057664,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.39.attn_q.weight",
"offset": 3436494848,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.39.attn_v.weight",
"offset": 3443703808,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.40.attn_norm.weight",
"offset": 3444883456,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 3444899840,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 3470262272,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.40.ffn_up.weight",
"offset": 3489636352,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 3509010432,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 3509026816,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.40.attn_output.weight",
"offset": 3509927936,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.40.attn_q.weight",
"offset": 3519365120,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.40.attn_v.weight",
"offset": 3526574080,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 3527753728,
"shape": [
4096,
11008
],
"size": 1179648,
"type": 11
},
{
"name": "blk.41.attn_k.weight",
"offset": 3547127808,
"shape": [
4096,
512
],
"size": 19374080,
"type": 11
},
{
"name": "blk.41.attn_output.weight",
"offset": 3548028928,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.41.attn_q.weight",
"offset": 3557466112,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.41.attn_v.weight",
"offset": 3564675072,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "output.weight",
"offset": 3565854720,
"shape": [
4096,
64000
],
"size": 1179648,
"type": 14
},
{
"name": "blk.41.attn_norm.weight",
"offset": 3780894720,
"shape": [
4096
],
"size": 215040000,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 3780911104,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.41.ffn_up.weight",
"offset": 3806273536,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 3825647616,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.42.attn_norm.weight",
"offset": 3825664000,
"shape": [
4096
],
"size": 16384,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 3825680384,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 3851042816,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.42.ffn_up.weight",
"offset": 3870416896,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 3889790976,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 3889807360,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.42.attn_output.weight",
"offset": 3890708480,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.42.attn_q.weight",
"offset": 3900145664,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.42.attn_v.weight",
"offset": 3907354624,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.43.attn_norm.weight",
"offset": 3908534272,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 3908550656,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 3933913088,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.43.ffn_up.weight",
"offset": 3953287168,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 3972661248,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 3972677632,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.43.attn_output.weight",
"offset": 3973578752,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.43.attn_q.weight",
"offset": 3983015936,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.43.attn_v.weight",
"offset": 3990224896,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.44.attn_norm.weight",
"offset": 3991404544,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 3991420928,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 4016783360,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.44.ffn_up.weight",
"offset": 4036157440,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 4055531520,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 4055547904,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.44.attn_output.weight",
"offset": 4056449024,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.44.attn_q.weight",
"offset": 4065886208,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.44.attn_v.weight",
"offset": 4073095168,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.45.attn_norm.weight",
"offset": 4074274816,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 4074291200,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 4099653632,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.45.ffn_up.weight",
"offset": 4119027712,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 4138401792,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 4138418176,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.45.attn_output.weight",
"offset": 4139319296,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.45.attn_q.weight",
"offset": 4148756480,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.45.attn_v.weight",
"offset": 4155965440,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.46.attn_norm.weight",
"offset": 4157145088,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 4157161472,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 4182523904,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.46.ffn_up.weight",
"offset": 4201897984,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 4221272064,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 4221288448,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.46.attn_output.weight",
"offset": 4222189568,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.46.attn_q.weight",
"offset": 4231626752,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.46.attn_v.weight",
"offset": 4238835712,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "blk.47.attn_norm.weight",
"offset": 4240015360,
"shape": [
4096
],
"size": 1179648,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 4240031744,
"shape": [
11008,
4096
],
"size": 16384,
"type": 12
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 4265394176,
"shape": [
4096,
11008
],
"size": 25362432,
"type": 11
},
{
"name": "blk.47.ffn_up.weight",
"offset": 4284768256,
"shape": [
4096,
11008
],
"size": 19374080,
"type": 11
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 4304142336,
"shape": [
4096
],
"size": 19374080,
"type": 0
},
{
"name": "\u0000\f\u0000\ufffd\ufffd\f\u0000\ufffd\u0000\u0000\ufffd\u0000\u0002\ufffd\u000e\ufffd\u001f\ufffd\ufffd\u001f",
"offset": 4304158720,
"shape": [
4096,
512
],
"size": 16384,
"type": 11
},
{
"name": "blk.47.attn_output.weight",
"offset": 4305059840,
"shape": [
4096,
4096
],
"size": 901120,
"type": 12
},
{
"name": "blk.47.attn_q.weight",
"offset": 4314497024,
"shape": [
4096,
4096
],
"size": 9437184,
"type": 11
},
{
"name": "blk.47.attn_v.weight",
"offset": 4321705984,
"shape": [
4096,
512
],
"size": 7208960,
"type": 12
},
{
"name": "output_norm.weight",
"offset": 4322885632,
"shape": [
4096
],
"size": 1179648,
"type": 0
}
],
"version": 3
}