Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
0ed36d85d262 · 24GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 17,
"general.name": "Yi-1.5-34B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 56,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 60,
"llama.context_length": 16384,
"llama.embedding_length": 7168,
"llama.feed_forward_length": 20480,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 34388917248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 315392000,
"shape": [
7168
],
"size": 315392000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 315420672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 435843072,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.0.ffn_up.weight",
"offset": 536768512,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 637693952,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 637722624,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 642768896,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.0.attn_q.weight",
"offset": 678092800,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.0.attn_v.weight",
"offset": 713416704,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 719437824,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 719466496,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 839888896,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.1.ffn_up.weight",
"offset": 940814336,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1041739776,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1041768448,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 1046814720,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.1.attn_q.weight",
"offset": 1082138624,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.1.attn_v.weight",
"offset": 1117462528,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1123483648,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1123512320,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1243934720,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1344860160,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1445785600,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 1445814272,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 1450860544,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.2.attn_q.weight",
"offset": 1486184448,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.2.attn_v.weight",
"offset": 1521508352,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1527529472,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 13
},
{
"name": "blk.3.attn_k.weight",
"offset": 1628454912,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 1633501184,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.3.attn_q.weight",
"offset": 1668825088,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.3.attn_v.weight",
"offset": 1704148992,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1710170112,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1710198784,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1830621184,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1931546624,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1931575296,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1931603968,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 2052026368,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.4.ffn_up.weight",
"offset": 2152951808,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 2253877248,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 2253905920,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 2258952192,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.4.attn_q.weight",
"offset": 2294276096,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.4.attn_v.weight",
"offset": 2329600000,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 2335621120,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 2335649792,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 2456072192,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.5.ffn_up.weight",
"offset": 2556997632,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 2657923072,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 2657951744,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 2662998016,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.5.attn_q.weight",
"offset": 2698321920,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.5.attn_v.weight",
"offset": 2733645824,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 2739666944,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 2739695616,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 2860118016,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2961043456,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 3061968896,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 3061997568,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 3067043840,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.6.attn_q.weight",
"offset": 3102367744,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.6.attn_v.weight",
"offset": 3137691648,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 3143712768,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 13
},
{
"name": "blk.7.ffn_up.weight",
"offset": 3244638208,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.7.attn_k.weight",
"offset": 3345563648,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 3350609920,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.7.attn_q.weight",
"offset": 3385933824,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.7.attn_v.weight",
"offset": 3421257728,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.10.attn_norm.weight",
"offset": 3426304000,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 3426332672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 3527258112,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.10.ffn_up.weight",
"offset": 3628183552,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 3729108992,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 3729137664,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 3734183936,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.10.attn_q.weight",
"offset": 3769507840,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.10.attn_v.weight",
"offset": 3804831744,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.11.attn_norm.weight",
"offset": 3809878016,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 3809906688,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 3910832128,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.11.ffn_up.weight",
"offset": 4011757568,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 4112683008,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 4112711680,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 4117757952,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.11.attn_q.weight",
"offset": 4153081856,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.11.attn_v.weight",
"offset": 4188405760,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.12.attn_k.weight",
"offset": 4194426880,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 4199473152,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.12.attn_q.weight",
"offset": 4234797056,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.12.attn_v.weight",
"offset": 4270120960,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.7.attn_norm.weight",
"offset": 4275167232,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 4275195904,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 4395618304,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 4395646976,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 4395675648,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 4496601088,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.8.ffn_up.weight",
"offset": 4597526528,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 4698451968,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 4698480640,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 4703526912,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.8.attn_q.weight",
"offset": 4738850816,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.8.attn_v.weight",
"offset": 4774174720,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.9.attn_norm.weight",
"offset": 4779220992,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 4779249664,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 4880175104,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.9.ffn_up.weight",
"offset": 4981100544,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 5082025984,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 5082054656,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 5087100928,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.9.attn_q.weight",
"offset": 5122424832,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.9.attn_v.weight",
"offset": 5157748736,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 5163769856,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 5163798528,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 5284220928,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.12.ffn_up.weight",
"offset": 5385146368,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 5486071808,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 5486100480,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 5486129152,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 5587054592,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.13.ffn_up.weight",
"offset": 5687980032,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 5788905472,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 5788934144,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 5793980416,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.13.attn_q.weight",
"offset": 5829304320,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.13.attn_v.weight",
"offset": 5864628224,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.14.attn_norm.weight",
"offset": 5869674496,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 5869703168,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 5970628608,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.14.ffn_up.weight",
"offset": 6071554048,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 6172479488,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 6172508160,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 6177554432,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.14.attn_q.weight",
"offset": 6212878336,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.14.attn_v.weight",
"offset": 6248202240,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.15.attn_norm.weight",
"offset": 6253248512,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 6253277184,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 6373699584,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.15.ffn_up.weight",
"offset": 6474625024,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 6575550464,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 6575579136,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 6580625408,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.15.attn_q.weight",
"offset": 6615949312,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.15.attn_v.weight",
"offset": 6651273216,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 6657294336,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 13
},
{
"name": "blk.16.attn_k.weight",
"offset": 6758219776,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 6763266048,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.16.attn_q.weight",
"offset": 6798589952,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.16.attn_v.weight",
"offset": 6833913856,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.16.attn_norm.weight",
"offset": 6838960128,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 6838988800,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.16.ffn_up.weight",
"offset": 6939914240,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 7040839680,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.17.attn_norm.weight",
"offset": 7040868352,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 7040897024,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 7141822464,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.17.ffn_up.weight",
"offset": 7242747904,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 7343673344,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 7343702016,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 7348748288,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.17.attn_q.weight",
"offset": 7384072192,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.17.attn_v.weight",
"offset": 7419396096,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.18.attn_norm.weight",
"offset": 7424442368,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 7424471040,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 7544893440,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.18.ffn_up.weight",
"offset": 7645818880,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 7746744320,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 7746772992,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 7751819264,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.18.attn_q.weight",
"offset": 7787143168,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.18.attn_v.weight",
"offset": 7822467072,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 7828488192,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 7828516864,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 7929442304,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.19.ffn_up.weight",
"offset": 8030367744,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 8131293184,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 8131321856,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 8136368128,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.19.attn_q.weight",
"offset": 8171692032,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.19.attn_v.weight",
"offset": 8207015936,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 8212062208,
"shape": [
7168,
20480
],
"size": 5046272,
"type": 13
},
{
"name": "blk.20.ffn_up.weight",
"offset": 8312987648,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.20.attn_k.weight",
"offset": 8413913088,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 8418959360,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.20.attn_q.weight",
"offset": 8454283264,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.20.attn_v.weight",
"offset": 8489607168,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.20.attn_norm.weight",
"offset": 8494653440,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 8494682112,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 8595607552,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 8595636224,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 8595664896,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 8716087296,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.21.ffn_up.weight",
"offset": 8817012736,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 8917938176,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 8917966848,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 8923013120,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.21.attn_q.weight",
"offset": 8958337024,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.21.attn_v.weight",
"offset": 8993660928,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 8999682048,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 8999710720,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 9100636160,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.22.ffn_up.weight",
"offset": 9201561600,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 9302487040,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 9302515712,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 9307561984,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.22.attn_q.weight",
"offset": 9342885888,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.22.attn_v.weight",
"offset": 9378209792,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.23.attn_norm.weight",
"offset": 9383256064,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 9383284736,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 9484210176,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.23.ffn_up.weight",
"offset": 9585135616,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 9686061056,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 9686089728,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 9691136000,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.23.attn_q.weight",
"offset": 9726459904,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.23.attn_v.weight",
"offset": 9761783808,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.24.attn_norm.weight",
"offset": 9766830080,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 9766858752,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 9887281152,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.24.ffn_up.weight",
"offset": 9988206592,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 10089132032,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 10089160704,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 10094206976,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.24.attn_q.weight",
"offset": 10129530880,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.24.attn_v.weight",
"offset": 10164854784,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.25.attn_k.weight",
"offset": 10170875904,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 10175922176,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.25.attn_q.weight",
"offset": 10211246080,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.25.attn_v.weight",
"offset": 10246569984,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.25.attn_norm.weight",
"offset": 10251616256,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 10251644928,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 10352570368,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.25.ffn_up.weight",
"offset": 10453495808,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 10554421248,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.26.attn_norm.weight",
"offset": 10554449920,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 10554478592,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 10655404032,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.26.ffn_up.weight",
"offset": 10756329472,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 10857254912,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 10857283584,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 10862329856,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.26.attn_q.weight",
"offset": 10897653760,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.26.attn_v.weight",
"offset": 10932977664,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 10938023936,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 10938052608,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 11058475008,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.27.ffn_up.weight",
"offset": 11159400448,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 11260325888,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 11260354560,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 11265400832,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.27.attn_q.weight",
"offset": 11300724736,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.27.attn_v.weight",
"offset": 11336048640,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 11342069760,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 11342098432,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 11443023872,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.28.ffn_up.weight",
"offset": 11543949312,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 11644874752,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 11644903424,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 11649949696,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.28.attn_q.weight",
"offset": 11685273600,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.28.attn_v.weight",
"offset": 11720597504,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 11725643776,
"shape": [
7168,
20480
],
"size": 5046272,
"type": 13
},
{
"name": "blk.29.attn_k.weight",
"offset": 11826569216,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 11831615488,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.29.attn_q.weight",
"offset": 11866939392,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.29.attn_v.weight",
"offset": 11902263296,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.29.attn_norm.weight",
"offset": 11907309568,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 11907338240,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.29.ffn_up.weight",
"offset": 12008263680,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 12109189120,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.30.attn_norm.weight",
"offset": 12109217792,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 12109246464,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 12229668864,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.30.ffn_up.weight",
"offset": 12330594304,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 12431519744,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 12431548416,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 12436594688,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.30.attn_q.weight",
"offset": 12471918592,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.30.attn_v.weight",
"offset": 12507242496,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 12513263616,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 12513292288,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 12614217728,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.31.ffn_up.weight",
"offset": 12715143168,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 12816068608,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 12816097280,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 12821143552,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.31.attn_q.weight",
"offset": 12856467456,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.31.attn_v.weight",
"offset": 12891791360,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.32.attn_norm.weight",
"offset": 12896837632,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 12896866304,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 12997791744,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.32.ffn_up.weight",
"offset": 13098717184,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 13199642624,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 13199671296,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.32.attn_output.weight",
"offset": 13204717568,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.32.attn_q.weight",
"offset": 13240041472,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.32.attn_v.weight",
"offset": 13275365376,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 13280411648,
"shape": [
7168,
20480
],
"size": 5046272,
"type": 13
},
{
"name": "blk.33.ffn_up.weight",
"offset": 13381337088,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.33.attn_k.weight",
"offset": 13482262528,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.33.attn_output.weight",
"offset": 13487308800,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.33.attn_q.weight",
"offset": 13522632704,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.33.attn_v.weight",
"offset": 13557956608,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 13563977728,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 13564006400,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 13684428800,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.34.attn_norm.weight",
"offset": 13684457472,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 13684486144,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 13785411584,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.34.ffn_up.weight",
"offset": 13886337024,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 13987262464,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 13987291136,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.34.attn_output.weight",
"offset": 13992337408,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.34.attn_q.weight",
"offset": 14027661312,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.34.attn_v.weight",
"offset": 14062985216,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.35.attn_norm.weight",
"offset": 14068031488,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 14068060160,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 14168985600,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.35.ffn_up.weight",
"offset": 14269911040,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 14370836480,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 14370865152,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.35.attn_output.weight",
"offset": 14375911424,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.35.attn_q.weight",
"offset": 14411235328,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.35.attn_v.weight",
"offset": 14446559232,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.36.attn_norm.weight",
"offset": 14451605504,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 14451634176,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 14572056576,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.36.ffn_up.weight",
"offset": 14672982016,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 14773907456,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 14773936128,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.36.attn_output.weight",
"offset": 14778982400,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.36.attn_q.weight",
"offset": 14814306304,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.36.attn_v.weight",
"offset": 14849630208,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 14855651328,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 14855680000,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 14956605440,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.37.ffn_up.weight",
"offset": 15057530880,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 15158456320,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 15158484992,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.37.attn_output.weight",
"offset": 15163531264,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.37.attn_q.weight",
"offset": 15198855168,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.37.attn_v.weight",
"offset": 15234179072,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.38.attn_k.weight",
"offset": 15239225344,
"shape": [
7168,
1024
],
"size": 5046272,
"type": 13
},
{
"name": "blk.38.attn_output.weight",
"offset": 15244271616,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.38.attn_q.weight",
"offset": 15279595520,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.38.attn_v.weight",
"offset": 15314919424,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.38.attn_norm.weight",
"offset": 15319965696,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 15319994368,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 15420919808,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.38.ffn_up.weight",
"offset": 15521845248,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 15622770688,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 15622799360,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 15622828032,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 15743250432,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.39.ffn_up.weight",
"offset": 15844175872,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 15945101312,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 15945129984,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.39.attn_output.weight",
"offset": 15950176256,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.39.attn_q.weight",
"offset": 15985500160,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.39.attn_v.weight",
"offset": 16020824064,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.40.attn_norm.weight",
"offset": 16026845184,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 16026873856,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 16127799296,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.40.ffn_up.weight",
"offset": 16228724736,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 16329650176,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 16329678848,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.40.attn_output.weight",
"offset": 16334725120,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.40.attn_q.weight",
"offset": 16370049024,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.40.attn_v.weight",
"offset": 16405372928,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.41.attn_norm.weight",
"offset": 16410419200,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 16410447872,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 16511373312,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.41.ffn_up.weight",
"offset": 16612298752,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 16713224192,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 16713252864,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.41.attn_output.weight",
"offset": 16718299136,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.41.attn_q.weight",
"offset": 16753623040,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.41.attn_v.weight",
"offset": 16788946944,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 16793993216,
"shape": [
7168,
20480
],
"size": 5046272,
"type": 13
},
{
"name": "blk.42.attn_k.weight",
"offset": 16894918656,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.42.attn_output.weight",
"offset": 16899964928,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.42.attn_q.weight",
"offset": 16935288832,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.42.attn_v.weight",
"offset": 16970612736,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 16976633856,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 16976662528,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.42.ffn_up.weight",
"offset": 17097084928,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 17198010368,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.43.attn_norm.weight",
"offset": 17198039040,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 17198067712,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 17298993152,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.43.ffn_up.weight",
"offset": 17399918592,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 17500844032,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 17500872704,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.43.attn_output.weight",
"offset": 17505918976,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.43.attn_q.weight",
"offset": 17541242880,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.43.attn_v.weight",
"offset": 17576566784,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.44.attn_norm.weight",
"offset": 17581613056,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 17581641728,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 17682567168,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.44.ffn_up.weight",
"offset": 17783492608,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 17884418048,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 17884446720,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.44.attn_output.weight",
"offset": 17889492992,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.44.attn_q.weight",
"offset": 17924816896,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.44.attn_v.weight",
"offset": 17960140800,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.45.attn_norm.weight",
"offset": 17965187072,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 17965215744,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 18085638144,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.45.ffn_up.weight",
"offset": 18186563584,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 18287489024,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 18287517696,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.45.attn_output.weight",
"offset": 18292563968,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.45.attn_q.weight",
"offset": 18327887872,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.45.attn_v.weight",
"offset": 18363211776,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 18369232896,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 13
},
{
"name": "blk.46.ffn_up.weight",
"offset": 18470158336,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.46.attn_k.weight",
"offset": 18571083776,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.46.attn_output.weight",
"offset": 18576130048,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.46.attn_q.weight",
"offset": 18611453952,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.46.attn_v.weight",
"offset": 18646777856,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.46.attn_norm.weight",
"offset": 18651824128,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 18651852800,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 18752778240,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.47.attn_norm.weight",
"offset": 18752806912,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 18752835584,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 18853761024,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.47.ffn_up.weight",
"offset": 18954686464,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 19055611904,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 19055640576,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "\u0000\u001c\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001c\u0000\u0000\u0000\u0000\u0000\u0000\r\u0000\u0000\u0000\u0000\u0000\u001bp\u0004",
"offset": 19060686848,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.47.attn_q.weight",
"offset": 19096010752,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.47.attn_v.weight",
"offset": 19131334656,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.48.attn_norm.weight",
"offset": 19136380928,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 19136409600,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 19256832000,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.48.ffn_up.weight",
"offset": 19357757440,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 19458682880,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 19458711552,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.48.attn_output.weight",
"offset": 19463757824,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.48.attn_q.weight",
"offset": 19499081728,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.48.attn_v.weight",
"offset": 19534405632,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.49.attn_norm.weight",
"offset": 19540426752,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 19540455424,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 19641380864,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.49.ffn_up.weight",
"offset": 19742306304,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 19843231744,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 19843260416,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.49.attn_output.weight",
"offset": 19848306688,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.49.attn_q.weight",
"offset": 19883630592,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.49.attn_v.weight",
"offset": 19918954496,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.50.attn_norm.weight",
"offset": 19924000768,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 19924029440,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 20024954880,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.50.ffn_up.weight",
"offset": 20125880320,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 20226805760,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 20226834432,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.50.attn_output.weight",
"offset": 20231880704,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.50.attn_q.weight",
"offset": 20267204608,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.50.attn_v.weight",
"offset": 20302528512,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 13
},
{
"name": "blk.51.attn_k.weight",
"offset": 20307574784,
"shape": [
7168,
1024
],
"size": 5046272,
"type": 13
},
{
"name": "blk.51.attn_output.weight",
"offset": 20312621056,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.51.attn_q.weight",
"offset": 20347944960,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.51.attn_v.weight",
"offset": 20383268864,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.51.attn_norm.weight",
"offset": 20389289984,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 20389318656,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 20509741056,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.51.ffn_up.weight",
"offset": 20610666496,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 20711591936,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.52.attn_norm.weight",
"offset": 20711620608,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 20711649280,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 20832071680,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.52.ffn_up.weight",
"offset": 20932997120,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 21033922560,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 21033951232,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.52.attn_output.weight",
"offset": 21038997504,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.52.attn_q.weight",
"offset": 21074321408,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.52.attn_v.weight",
"offset": 21109645312,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.53.attn_norm.weight",
"offset": 21115666432,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 21115695104,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 21236117504,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.53.ffn_up.weight",
"offset": 21337042944,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 21437968384,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 21437997056,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.53.attn_output.weight",
"offset": 21443043328,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.53.attn_q.weight",
"offset": 21478367232,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.53.attn_v.weight",
"offset": 21513691136,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.54.attn_norm.weight",
"offset": 21519712256,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 21519740928,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 21640163328,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.54.ffn_up.weight",
"offset": 21741088768,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 21842014208,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 21842042880,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.54.attn_output.weight",
"offset": 21847089152,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.54.attn_q.weight",
"offset": 21882413056,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.54.attn_v.weight",
"offset": 21917736960,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 21923758080,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 13
},
{
"name": "blk.55.attn_k.weight",
"offset": 22024683520,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.55.attn_output.weight",
"offset": 22029729792,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.55.attn_q.weight",
"offset": 22065053696,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.55.attn_v.weight",
"offset": 22100377600,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.55.attn_norm.weight",
"offset": 22106398720,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 22106427392,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.55.ffn_up.weight",
"offset": 22226849792,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 22327775232,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.56.attn_norm.weight",
"offset": 22327803904,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 22327832576,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 22448254976,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.56.ffn_up.weight",
"offset": 22549180416,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 22650105856,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 22650134528,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.56.attn_output.weight",
"offset": 22655180800,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.56.attn_q.weight",
"offset": 22690504704,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.56.attn_v.weight",
"offset": 22725828608,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.57.attn_norm.weight",
"offset": 22731849728,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 22731878400,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 22852300800,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.57.ffn_up.weight",
"offset": 22953226240,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 23054151680,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 23054180352,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.57.attn_output.weight",
"offset": 23059226624,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.57.attn_q.weight",
"offset": 23094550528,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.57.attn_v.weight",
"offset": 23129874432,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.58.attn_norm.weight",
"offset": 23135895552,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 23135924224,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 23256346624,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 13
},
{
"name": "blk.58.ffn_up.weight",
"offset": 23357272064,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 23458197504,
"shape": [
7168
],
"size": 100925440,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 23458226176,
"shape": [
7168,
1024
],
"size": 28672,
"type": 13
},
{
"name": "blk.58.attn_output.weight",
"offset": 23463272448,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.58.attn_q.weight",
"offset": 23498596352,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.58.attn_v.weight",
"offset": 23533920256,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 23539941376,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 13
},
{
"name": "blk.59.ffn_up.weight",
"offset": 23640866816,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 13
},
{
"name": "blk.59.attn_k.weight",
"offset": 23741792256,
"shape": [
7168,
1024
],
"size": 100925440,
"type": 13
},
{
"name": "blk.59.attn_output.weight",
"offset": 23746838528,
"shape": [
7168,
7168
],
"size": 5046272,
"type": 13
},
{
"name": "blk.59.attn_q.weight",
"offset": 23782162432,
"shape": [
7168,
7168
],
"size": 35323904,
"type": 13
},
{
"name": "blk.59.attn_v.weight",
"offset": 23817486336,
"shape": [
7168,
1024
],
"size": 35323904,
"type": 14
},
{
"name": "output.weight",
"offset": 23823507456,
"shape": [
7168,
64000
],
"size": 6021120,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 24199827456,
"shape": [
7168
],
"size": 376320000,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 24199856128,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 24320278528,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 24320307200,
"shape": [
7168
],
"size": 28672,
"type": 0
}
],
"version": 3
}