Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
4b531ef46209 · 17GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 12,
"general.name": "Yi-1.5-34B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 56,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 60,
"llama.context_length": 16384,
"llama.embedding_length": 7168,
"llama.feed_forward_length": 20480,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 34388917248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 197120000,
"shape": [
7168
],
"size": 197120000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 197148672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 298074112,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 11
},
{
"name": "blk.0.ffn_up.weight",
"offset": 361152512,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 424230912,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 424259584,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.0.attn_output.weight",
"offset": 427413504,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 456314880,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.0.attn_v.weight",
"offset": 478392320,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 13
},
{
"name": "blk.1.attn_norm.weight",
"offset": 483438592,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 483467264,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 584392704,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 11
},
{
"name": "blk.1.ffn_up.weight",
"offset": 647471104,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 710549504,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 710578176,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.1.attn_output.weight",
"offset": 713732096,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 742633472,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.1.attn_v.weight",
"offset": 764710912,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 13
},
{
"name": "blk.2.attn_norm.weight",
"offset": 769757184,
"shape": [
7168
],
"size": 5046272,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 769785856,
"shape": [
20480,
7168
],
"size": 28672,
"type": 13
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 870711296,
"shape": [
7168,
20480
],
"size": 100925440,
"type": 11
},
{
"name": "blk.2.ffn_up.weight",
"offset": 933789696,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 996868096,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 996896768,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.2.attn_output.weight",
"offset": 1000050688,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 1028952064,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.2.attn_v.weight",
"offset": 1051029504,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1055158272,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.3.attn_k.weight",
"offset": 1118236672,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.3.attn_output.weight",
"offset": 1121390592,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 1150291968,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.3.attn_v.weight",
"offset": 1172369408,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1176498176,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1176526848,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1259102208,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1322180608,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1322209280,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1322237952,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1404813312,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1467891712,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1530970112,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1530998784,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.4.attn_output.weight",
"offset": 1534152704,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 1563054080,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.4.attn_v.weight",
"offset": 1585131520,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1589260288,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1589288960,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1671864320,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1734942720,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1798021120,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1798049792,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.5.attn_output.weight",
"offset": 1801203712,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 1830105088,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.5.attn_v.weight",
"offset": 1852182528,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1856311296,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1856339968,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1938915328,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2001993728,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 2065072128,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 2065100800,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.6.attn_output.weight",
"offset": 2068254720,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 2097156096,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.6.attn_v.weight",
"offset": 2119233536,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 2123362304,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.7.ffn_up.weight",
"offset": 2186440704,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.7.attn_k.weight",
"offset": 2249519104,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.7.attn_output.weight",
"offset": 2252673024,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 2281574400,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.7.attn_v.weight",
"offset": 2303651840,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2307780608,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2307809280,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 2390384640,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.10.ffn_up.weight",
"offset": 2453463040,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 2516541440,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 2516570112,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.10.attn_output.weight",
"offset": 2519724032,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 2548625408,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.10.attn_v.weight",
"offset": 2570702848,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 2574831616,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 2574860288,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 2657435648,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.11.ffn_up.weight",
"offset": 2720514048,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 2783592448,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 2783621120,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.11.attn_output.weight",
"offset": 2786775040,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 2815676416,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.11.attn_v.weight",
"offset": 2837753856,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.12.attn_k.weight",
"offset": 2841882624,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 11
},
{
"name": "blk.12.attn_output.weight",
"offset": 2845036544,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 2873937920,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.12.attn_v.weight",
"offset": 2896015360,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2900144128,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2900172800,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 2982748160,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 2982776832,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 2982805504,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 3065380864,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.8.ffn_up.weight",
"offset": 3128459264,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 3191537664,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 3191566336,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.8.attn_output.weight",
"offset": 3194720256,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 3223621632,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.8.attn_v.weight",
"offset": 3245699072,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3249827840,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3249856512,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3332431872,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3395510272,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3458588672,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 3458617344,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.9.attn_output.weight",
"offset": 3461771264,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 3490672640,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.9.attn_v.weight",
"offset": 3512750080,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 3516878848,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 3516907520,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 3599482880,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.12.ffn_up.weight",
"offset": 3662561280,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 3725639680,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 3725668352,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 3725697024,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 3808272384,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.13.ffn_up.weight",
"offset": 3871350784,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 3934429184,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 3934457856,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.13.attn_output.weight",
"offset": 3937611776,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 3966513152,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.13.attn_v.weight",
"offset": 3988590592,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 3992719360,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 3992748032,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 4075323392,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.14.ffn_up.weight",
"offset": 4138401792,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 4201480192,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 4201508864,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.14.attn_output.weight",
"offset": 4204662784,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 4233564160,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.14.attn_v.weight",
"offset": 4255641600,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 4259770368,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 4259799040,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 4342374400,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.15.ffn_up.weight",
"offset": 4405452800,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 4468531200,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 4468559872,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.15.attn_output.weight",
"offset": 4471713792,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 4500615168,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.15.attn_v.weight",
"offset": 4522692608,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 4526821376,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.16.attn_k.weight",
"offset": 4589899776,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.16.attn_output.weight",
"offset": 4593053696,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 4621955072,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.16.attn_v.weight",
"offset": 4644032512,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 4648161280,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 4648189952,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 4730765312,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 4793843712,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.17.attn_norm.weight",
"offset": 4793872384,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 4793901056,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 4876476416,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.17.ffn_up.weight",
"offset": 4939554816,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 5002633216,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 5002661888,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.17.attn_output.weight",
"offset": 5005815808,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 5034717184,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.17.attn_v.weight",
"offset": 5056794624,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 5060923392,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 5060952064,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 5143527424,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.18.ffn_up.weight",
"offset": 5206605824,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 5269684224,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 5269712896,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.18.attn_output.weight",
"offset": 5272866816,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 5301768192,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.18.attn_v.weight",
"offset": 5323845632,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 5327974400,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 5328003072,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 5410578432,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.19.ffn_up.weight",
"offset": 5473656832,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 5536735232,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 5536763904,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.19.attn_output.weight",
"offset": 5539917824,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 5568819200,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.19.attn_v.weight",
"offset": 5590896640,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 5595025408,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.20.ffn_up.weight",
"offset": 5658103808,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.20.attn_k.weight",
"offset": 5721182208,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.20.attn_output.weight",
"offset": 5724336128,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 5753237504,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.20.attn_v.weight",
"offset": 5775314944,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 5779443712,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 5779472384,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 5862047744,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 5862076416,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 5862105088,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 5944680448,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.21.ffn_up.weight",
"offset": 6007758848,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 6070837248,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 6070865920,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.21.attn_output.weight",
"offset": 6074019840,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 6102921216,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.21.attn_v.weight",
"offset": 6124998656,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 6129127424,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 6129156096,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 6211731456,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.22.ffn_up.weight",
"offset": 6274809856,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 6337888256,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 6337916928,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.22.attn_output.weight",
"offset": 6341070848,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 6369972224,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.22.attn_v.weight",
"offset": 6392049664,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 6396178432,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 6396207104,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 6478782464,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.23.ffn_up.weight",
"offset": 6541860864,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 6604939264,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 6604967936,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.23.attn_output.weight",
"offset": 6608121856,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 6637023232,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.23.attn_v.weight",
"offset": 6659100672,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 6663229440,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 6663258112,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 6745833472,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.24.ffn_up.weight",
"offset": 6808911872,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 6871990272,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 6872018944,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.24.attn_output.weight",
"offset": 6875172864,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 6904074240,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.24.attn_v.weight",
"offset": 6926151680,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.25.attn_k.weight",
"offset": 6930280448,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 11
},
{
"name": "blk.25.attn_output.weight",
"offset": 6933434368,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 6962335744,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.25.attn_v.weight",
"offset": 6984413184,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 6988541952,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 6988570624,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 7071145984,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.25.ffn_up.weight",
"offset": 7134224384,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 7197302784,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.26.attn_norm.weight",
"offset": 7197331456,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 7197360128,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 7279935488,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.26.ffn_up.weight",
"offset": 7343013888,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 7406092288,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 7406120960,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.26.attn_output.weight",
"offset": 7409274880,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 7438176256,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.26.attn_v.weight",
"offset": 7460253696,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 7464382464,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 7464411136,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 7546986496,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.27.ffn_up.weight",
"offset": 7610064896,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 7673143296,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 7673171968,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.27.attn_output.weight",
"offset": 7676325888,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 7705227264,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.27.attn_v.weight",
"offset": 7727304704,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 7731433472,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 7731462144,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 7814037504,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.28.ffn_up.weight",
"offset": 7877115904,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 7940194304,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 7940222976,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.28.attn_output.weight",
"offset": 7943376896,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 7972278272,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.28.attn_v.weight",
"offset": 7994355712,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 7998484480,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.29.attn_k.weight",
"offset": 8061562880,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.29.attn_output.weight",
"offset": 8064716800,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 8093618176,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.29.attn_v.weight",
"offset": 8115695616,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 8119824384,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 8119853056,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 8202428416,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 8265506816,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.30.attn_norm.weight",
"offset": 8265535488,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 8265564160,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 8348139520,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.30.ffn_up.weight",
"offset": 8411217920,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 8474296320,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 8474324992,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.30.attn_output.weight",
"offset": 8477478912,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 8506380288,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.30.attn_v.weight",
"offset": 8528457728,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.31.attn_norm.weight",
"offset": 8532586496,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 8532615168,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 8615190528,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.31.ffn_up.weight",
"offset": 8678268928,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 8741347328,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 8741376000,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.31.attn_output.weight",
"offset": 8744529920,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 8773431296,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.31.attn_v.weight",
"offset": 8795508736,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.32.attn_norm.weight",
"offset": 8799637504,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 8799666176,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 8882241536,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.32.ffn_up.weight",
"offset": 8945319936,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 9008398336,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 9008427008,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.32.attn_output.weight",
"offset": 9011580928,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.32.attn_q.weight",
"offset": 9040482304,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.32.attn_v.weight",
"offset": 9062559744,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 9066688512,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.33.ffn_up.weight",
"offset": 9129766912,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.33.attn_k.weight",
"offset": 9192845312,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.33.attn_output.weight",
"offset": 9195999232,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.33.attn_q.weight",
"offset": 9224900608,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.33.attn_v.weight",
"offset": 9246978048,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.33.attn_norm.weight",
"offset": 9251106816,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 9251135488,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 9333710848,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.34.attn_norm.weight",
"offset": 9333739520,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 9333768192,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 9416343552,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.34.ffn_up.weight",
"offset": 9479421952,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 9542500352,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 9542529024,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.34.attn_output.weight",
"offset": 9545682944,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.34.attn_q.weight",
"offset": 9574584320,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.34.attn_v.weight",
"offset": 9596661760,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.35.attn_norm.weight",
"offset": 9600790528,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 9600819200,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 9683394560,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.35.ffn_up.weight",
"offset": 9746472960,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 9809551360,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 9809580032,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.35.attn_output.weight",
"offset": 9812733952,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.35.attn_q.weight",
"offset": 9841635328,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.35.attn_v.weight",
"offset": 9863712768,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.36.attn_norm.weight",
"offset": 9867841536,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 9867870208,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 9950445568,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.36.ffn_up.weight",
"offset": 10013523968,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 10076602368,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 10076631040,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.36.attn_output.weight",
"offset": 10079784960,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.36.attn_q.weight",
"offset": 10108686336,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.36.attn_v.weight",
"offset": 10130763776,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.37.attn_norm.weight",
"offset": 10134892544,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 10134921216,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 10217496576,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.37.ffn_up.weight",
"offset": 10280574976,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 10343653376,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 10343682048,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.37.attn_output.weight",
"offset": 10346835968,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.37.attn_q.weight",
"offset": 10375737344,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.37.attn_v.weight",
"offset": 10397814784,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.38.attn_k.weight",
"offset": 10401943552,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 11
},
{
"name": "blk.38.attn_output.weight",
"offset": 10405097472,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.38.attn_q.weight",
"offset": 10433998848,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.38.attn_v.weight",
"offset": 10456076288,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.38.attn_norm.weight",
"offset": 10460205056,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 10460233728,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 10542809088,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.38.ffn_up.weight",
"offset": 10605887488,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 10668965888,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 10668994560,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 10669023232,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 10751598592,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.39.ffn_up.weight",
"offset": 10814676992,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 10877755392,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 10877784064,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.39.attn_output.weight",
"offset": 10880937984,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.39.attn_q.weight",
"offset": 10909839360,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.39.attn_v.weight",
"offset": 10931916800,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.40.attn_norm.weight",
"offset": 10936045568,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 10936074240,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 11018649600,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.40.ffn_up.weight",
"offset": 11081728000,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 11144806400,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 11144835072,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.40.attn_output.weight",
"offset": 11147988992,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.40.attn_q.weight",
"offset": 11176890368,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.40.attn_v.weight",
"offset": 11198967808,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.41.attn_norm.weight",
"offset": 11203096576,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 11203125248,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 11285700608,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.41.ffn_up.weight",
"offset": 11348779008,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 11411857408,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 11411886080,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.41.attn_output.weight",
"offset": 11415040000,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.41.attn_q.weight",
"offset": 11443941376,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.41.attn_v.weight",
"offset": 11466018816,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 11470147584,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.42.attn_k.weight",
"offset": 11533225984,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.42.attn_output.weight",
"offset": 11536379904,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.42.attn_q.weight",
"offset": 11565281280,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.42.attn_v.weight",
"offset": 11587358720,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.42.attn_norm.weight",
"offset": 11591487488,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 11591516160,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.42.ffn_up.weight",
"offset": 11674091520,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 11737169920,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.43.attn_norm.weight",
"offset": 11737198592,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 11737227264,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 11819802624,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.43.ffn_up.weight",
"offset": 11882881024,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 11945959424,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 11945988096,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.43.attn_output.weight",
"offset": 11949142016,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.43.attn_q.weight",
"offset": 11978043392,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.43.attn_v.weight",
"offset": 12000120832,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.44.attn_norm.weight",
"offset": 12004249600,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 12004278272,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 12086853632,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.44.ffn_up.weight",
"offset": 12149932032,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 12213010432,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 12213039104,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.44.attn_output.weight",
"offset": 12216193024,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.44.attn_q.weight",
"offset": 12245094400,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.44.attn_v.weight",
"offset": 12267171840,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.45.attn_norm.weight",
"offset": 12271300608,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 12271329280,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 12353904640,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.45.ffn_up.weight",
"offset": 12416983040,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 12480061440,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 12480090112,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.45.attn_output.weight",
"offset": 12483244032,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.45.attn_q.weight",
"offset": 12512145408,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.45.attn_v.weight",
"offset": 12534222848,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 12538351616,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.46.ffn_up.weight",
"offset": 12601430016,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.46.attn_k.weight",
"offset": 12664508416,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.46.attn_output.weight",
"offset": 12667662336,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.46.attn_q.weight",
"offset": 12696563712,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.46.attn_v.weight",
"offset": 12718641152,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.46.attn_norm.weight",
"offset": 12722769920,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 12722798592,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 12805373952,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "blk.47.attn_norm.weight",
"offset": 12805402624,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 12805431296,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 12888006656,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.47.ffn_up.weight",
"offset": 12951085056,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 13014163456,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 13014192128,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.47.attn_output.weight",
"offset": 13017346048,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.47.attn_q.weight",
"offset": 13046247424,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.47.attn_v.weight",
"offset": 13068324864,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.48.attn_norm.weight",
"offset": 13072453632,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 13072482304,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 13155057664,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.48.ffn_up.weight",
"offset": 13218136064,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 13281214464,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 13281243136,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.48.attn_output.weight",
"offset": 13284397056,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.48.attn_q.weight",
"offset": 13313298432,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.48.attn_v.weight",
"offset": 13335375872,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.49.attn_norm.weight",
"offset": 13339504640,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 13339533312,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 13422108672,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.49.ffn_up.weight",
"offset": 13485187072,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 13548265472,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 13548294144,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.49.attn_output.weight",
"offset": 13551448064,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.49.attn_q.weight",
"offset": 13580349440,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.49.attn_v.weight",
"offset": 13602426880,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.50.attn_norm.weight",
"offset": 13606555648,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 13606584320,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 13689159680,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.50.ffn_up.weight",
"offset": 13752238080,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 13815316480,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 13815345152,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.50.attn_output.weight",
"offset": 13818499072,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.50.attn_q.weight",
"offset": 13847400448,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.50.attn_v.weight",
"offset": 13869477888,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.51.attn_k.weight",
"offset": 13873606656,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 11
},
{
"name": "blk.51.attn_output.weight",
"offset": 13876760576,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.51.attn_q.weight",
"offset": 13905661952,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.51.attn_v.weight",
"offset": 13927739392,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.51.attn_norm.weight",
"offset": 13931868160,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 13931896832,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 14014472192,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.51.ffn_up.weight",
"offset": 14077550592,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 14140628992,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.52.attn_norm.weight",
"offset": 14140657664,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 14140686336,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 14223261696,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.52.ffn_up.weight",
"offset": 14286340096,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 14349418496,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 14349447168,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.52.attn_output.weight",
"offset": 14352601088,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.52.attn_q.weight",
"offset": 14381502464,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.52.attn_v.weight",
"offset": 14403579904,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.53.attn_norm.weight",
"offset": 14407708672,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 14407737344,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 14490312704,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.53.ffn_up.weight",
"offset": 14553391104,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 14616469504,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 14616498176,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.53.attn_output.weight",
"offset": 14619652096,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.53.attn_q.weight",
"offset": 14648553472,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.53.attn_v.weight",
"offset": 14670630912,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.54.attn_norm.weight",
"offset": 14674759680,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 14674788352,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 14757363712,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.54.ffn_up.weight",
"offset": 14820442112,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 14883520512,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 14883549184,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.54.attn_output.weight",
"offset": 14886703104,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.54.attn_q.weight",
"offset": 14915604480,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.54.attn_v.weight",
"offset": 14937681920,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 14941810688,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.55.attn_k.weight",
"offset": 15004889088,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.55.attn_output.weight",
"offset": 15008043008,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.55.attn_q.weight",
"offset": 15036944384,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.55.attn_v.weight",
"offset": 15059021824,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.55.attn_norm.weight",
"offset": 15063150592,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 15063179264,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.55.ffn_up.weight",
"offset": 15145754624,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 15208833024,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.56.attn_norm.weight",
"offset": 15208861696,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 15208890368,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 15291465728,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.56.ffn_up.weight",
"offset": 15354544128,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 15417622528,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 15417651200,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.56.attn_output.weight",
"offset": 15420805120,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.56.attn_q.weight",
"offset": 15449706496,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.56.attn_v.weight",
"offset": 15471783936,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.57.attn_norm.weight",
"offset": 15475912704,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 15475941376,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 15558516736,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.57.ffn_up.weight",
"offset": 15621595136,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 15684673536,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 15684702208,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.57.attn_output.weight",
"offset": 15687856128,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.57.attn_q.weight",
"offset": 15716757504,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.57.attn_v.weight",
"offset": 15738834944,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.58.attn_norm.weight",
"offset": 15742963712,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 15742992384,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 15825567744,
"shape": [
7168,
20480
],
"size": 82575360,
"type": 11
},
{
"name": "blk.58.ffn_up.weight",
"offset": 15888646144,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 15951724544,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 15951753216,
"shape": [
7168,
1024
],
"size": 28672,
"type": 11
},
{
"name": "blk.58.attn_output.weight",
"offset": 15954907136,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.58.attn_q.weight",
"offset": 15983808512,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.58.attn_v.weight",
"offset": 16005885952,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 16010014720,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 11
},
{
"name": "blk.59.ffn_up.weight",
"offset": 16073093120,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 11
},
{
"name": "blk.59.attn_k.weight",
"offset": 16136171520,
"shape": [
7168,
1024
],
"size": 63078400,
"type": 11
},
{
"name": "blk.59.attn_output.weight",
"offset": 16139325440,
"shape": [
7168,
7168
],
"size": 3153920,
"type": 12
},
{
"name": "blk.59.attn_q.weight",
"offset": 16168226816,
"shape": [
7168,
7168
],
"size": 28901376,
"type": 11
},
{
"name": "blk.59.attn_v.weight",
"offset": 16190304256,
"shape": [
7168,
1024
],
"size": 22077440,
"type": 12
},
{
"name": "output.weight",
"offset": 16194433024,
"shape": [
7168,
64000
],
"size": 4128768,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 16570753024,
"shape": [
7168
],
"size": 376320000,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 16570781696,
"shape": [
20480,
7168
],
"size": 28672,
"type": 12
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 16653357056,
"shape": [
7168
],
"size": 82575360,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 16653385728,
"shape": [
7168
],
"size": 28672,
"type": 0
}
],
"version": 3
}