Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
aa727e2caf14 · 69GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 1,
"general.name": "Yi-1.5-34B-Chat-16K",
"llama.attention.head_count": 56,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 60,
"llama.context_length": 16384,
"llama.embedding_length": 7168,
"llama.feed_forward_length": 20480,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 34388917248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 917504000,
"shape": [
7168
],
"size": 917504000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 917532672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 1211133952,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1504735232,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1798336512,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 1798365184,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.0.attn_output.weight",
"offset": 1813045248,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.0.attn_q.weight",
"offset": 1915805696,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.0.attn_v.weight",
"offset": 2018566144,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2033246208,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 2033274880,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 2326876160,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.1.ffn_up.weight",
"offset": 2620477440,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 2914078720,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 2914107392,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.1.attn_output.weight",
"offset": 2928787456,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.1.attn_q.weight",
"offset": 3031547904,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.1.attn_v.weight",
"offset": 3134308352,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.2.attn_norm.weight",
"offset": 3148988416,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 3149017088,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 3442618368,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.2.ffn_up.weight",
"offset": 3736219648,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 4029820928,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 4029849600,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.2.attn_output.weight",
"offset": 4044529664,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.2.attn_q.weight",
"offset": 4147290112,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.2.attn_v.weight",
"offset": 4250050560,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 4264730624,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.3.attn_k.weight",
"offset": 4558331904,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.3.attn_output.weight",
"offset": 4573011968,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.3.attn_q.weight",
"offset": 4675772416,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.3.attn_v.weight",
"offset": 4778532864,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.3.attn_norm.weight",
"offset": 4793212928,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 4793241600,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.3.ffn_up.weight",
"offset": 5086842880,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 5380444160,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.4.attn_norm.weight",
"offset": 5380472832,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 5380501504,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 5674102784,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.4.ffn_up.weight",
"offset": 5967704064,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 6261305344,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 6261334016,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.4.attn_output.weight",
"offset": 6276014080,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.4.attn_q.weight",
"offset": 6378774528,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.4.attn_v.weight",
"offset": 6481534976,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.5.attn_norm.weight",
"offset": 6496215040,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 6496243712,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 6789844992,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.5.ffn_up.weight",
"offset": 7083446272,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 7377047552,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 7377076224,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.5.attn_output.weight",
"offset": 7391756288,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.5.attn_q.weight",
"offset": 7494516736,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.5.attn_v.weight",
"offset": 7597277184,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.6.attn_norm.weight",
"offset": 7611957248,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 7611985920,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 7905587200,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.6.ffn_up.weight",
"offset": 8199188480,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 8492789760,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 8492818432,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.6.attn_output.weight",
"offset": 8507498496,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.6.attn_q.weight",
"offset": 8610258944,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.6.attn_v.weight",
"offset": 8713019392,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 8727699456,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.7.ffn_up.weight",
"offset": 9021300736,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.7.attn_k.weight",
"offset": 9314902016,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.7.attn_output.weight",
"offset": 9329582080,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.7.attn_q.weight",
"offset": 9432342528,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.7.attn_v.weight",
"offset": 9535102976,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.10.attn_norm.weight",
"offset": 9549783040,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 9549811712,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 9843412992,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.10.ffn_up.weight",
"offset": 10137014272,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 10430615552,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 10430644224,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.10.attn_output.weight",
"offset": 10445324288,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.10.attn_q.weight",
"offset": 10548084736,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.10.attn_v.weight",
"offset": 10650845184,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.11.attn_norm.weight",
"offset": 10665525248,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 10665553920,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 10959155200,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.11.ffn_up.weight",
"offset": 11252756480,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 11546357760,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 11546386432,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.11.attn_output.weight",
"offset": 11561066496,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.11.attn_q.weight",
"offset": 11663826944,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.11.attn_v.weight",
"offset": 11766587392,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.12.attn_k.weight",
"offset": 11781267456,
"shape": [
7168,
1024
],
"size": 14680064,
"type": 1
},
{
"name": "blk.12.attn_output.weight",
"offset": 11795947520,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.12.attn_q.weight",
"offset": 11898707968,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.12.attn_v.weight",
"offset": 12001468416,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.7.attn_norm.weight",
"offset": 12016148480,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 12016177152,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 12309778432,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 12309807104,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 12309835776,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 12603437056,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.8.ffn_up.weight",
"offset": 12897038336,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 13190639616,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 13190668288,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.8.attn_output.weight",
"offset": 13205348352,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.8.attn_q.weight",
"offset": 13308108800,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.8.attn_v.weight",
"offset": 13410869248,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.9.attn_norm.weight",
"offset": 13425549312,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 13425577984,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 13719179264,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.9.ffn_up.weight",
"offset": 14012780544,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 14306381824,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 14306410496,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.9.attn_output.weight",
"offset": 14321090560,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.9.attn_q.weight",
"offset": 14423851008,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.9.attn_v.weight",
"offset": 14526611456,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.12.attn_norm.weight",
"offset": 14541291520,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 14541320192,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 14834921472,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.12.ffn_up.weight",
"offset": 15128522752,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 15422124032,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 15422152704,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 15422181376,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 15715782656,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.13.ffn_up.weight",
"offset": 16009383936,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 16302985216,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 16303013888,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.13.attn_output.weight",
"offset": 16317693952,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.13.attn_q.weight",
"offset": 16420454400,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.13.attn_v.weight",
"offset": 16523214848,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.14.attn_norm.weight",
"offset": 16537894912,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 16537923584,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 16831524864,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.14.ffn_up.weight",
"offset": 17125126144,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 17418727424,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 17418756096,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.14.attn_output.weight",
"offset": 17433436160,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.14.attn_q.weight",
"offset": 17536196608,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.14.attn_v.weight",
"offset": 17638957056,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.15.attn_norm.weight",
"offset": 17653637120,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 17653665792,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 17947267072,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.15.ffn_up.weight",
"offset": 18240868352,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 18534469632,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 18534498304,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.15.attn_output.weight",
"offset": 18549178368,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.15.attn_q.weight",
"offset": 18651938816,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.15.attn_v.weight",
"offset": 18754699264,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 18769379328,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.16.attn_k.weight",
"offset": 19062980608,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.16.attn_output.weight",
"offset": 19077660672,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.16.attn_q.weight",
"offset": 19180421120,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.16.attn_v.weight",
"offset": 19283181568,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.16.attn_norm.weight",
"offset": 19297861632,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 19297890304,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.16.ffn_up.weight",
"offset": 19591491584,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 19885092864,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.17.attn_norm.weight",
"offset": 19885121536,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 19885150208,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 20178751488,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.17.ffn_up.weight",
"offset": 20472352768,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 20765954048,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 20765982720,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.17.attn_output.weight",
"offset": 20780662784,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.17.attn_q.weight",
"offset": 20883423232,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.17.attn_v.weight",
"offset": 20986183680,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.18.attn_norm.weight",
"offset": 21000863744,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 21000892416,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 21294493696,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.18.ffn_up.weight",
"offset": 21588094976,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 21881696256,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 21881724928,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.18.attn_output.weight",
"offset": 21896404992,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.18.attn_q.weight",
"offset": 21999165440,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.18.attn_v.weight",
"offset": 22101925888,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.19.attn_norm.weight",
"offset": 22116605952,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 22116634624,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 22410235904,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.19.ffn_up.weight",
"offset": 22703837184,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 22997438464,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 22997467136,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.19.attn_output.weight",
"offset": 23012147200,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.19.attn_q.weight",
"offset": 23114907648,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.19.attn_v.weight",
"offset": 23217668096,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 23232348160,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.20.ffn_up.weight",
"offset": 23525949440,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.20.attn_k.weight",
"offset": 23819550720,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.20.attn_output.weight",
"offset": 23834230784,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.20.attn_q.weight",
"offset": 23936991232,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.20.attn_v.weight",
"offset": 24039751680,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.20.attn_norm.weight",
"offset": 24054431744,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 24054460416,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 24348061696,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 24348090368,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 24348119040,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 24641720320,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.21.ffn_up.weight",
"offset": 24935321600,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 25228922880,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 25228951552,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.21.attn_output.weight",
"offset": 25243631616,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.21.attn_q.weight",
"offset": 25346392064,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.21.attn_v.weight",
"offset": 25449152512,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.22.attn_norm.weight",
"offset": 25463832576,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 25463861248,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 25757462528,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.22.ffn_up.weight",
"offset": 26051063808,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 26344665088,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 26344693760,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.22.attn_output.weight",
"offset": 26359373824,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.22.attn_q.weight",
"offset": 26462134272,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.22.attn_v.weight",
"offset": 26564894720,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.23.attn_norm.weight",
"offset": 26579574784,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 26579603456,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 26873204736,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.23.ffn_up.weight",
"offset": 27166806016,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 27460407296,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 27460435968,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.23.attn_output.weight",
"offset": 27475116032,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.23.attn_q.weight",
"offset": 27577876480,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.23.attn_v.weight",
"offset": 27680636928,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.24.attn_norm.weight",
"offset": 27695316992,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 27695345664,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 27988946944,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.24.ffn_up.weight",
"offset": 28282548224,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 28576149504,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 28576178176,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.24.attn_output.weight",
"offset": 28590858240,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.24.attn_q.weight",
"offset": 28693618688,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "ffn_down.weight\u0002\u0000\u0000\u0000\u0000",
"offset": 28796379136,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.25.attn_k.weight",
"offset": 28811059200,
"shape": [
7168,
1024
],
"size": 14680064,
"type": 1
},
{
"name": "blk.25.attn_output.weight",
"offset": 28825739264,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.25.attn_q.weight",
"offset": 28928499712,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.25.attn_v.weight",
"offset": 29031260160,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.25.attn_norm.weight",
"offset": 29045940224,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 29045968896,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 29339570176,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.25.ffn_up.weight",
"offset": 29633171456,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 29926772736,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.26.attn_norm.weight",
"offset": 29926801408,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 29926830080,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 30220431360,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.26.ffn_up.weight",
"offset": 30514032640,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 30807633920,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 30807662592,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.26.attn_output.weight",
"offset": 30822342656,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.26.attn_q.weight",
"offset": 30925103104,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.26.attn_v.weight",
"offset": 31027863552,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.27.attn_norm.weight",
"offset": 31042543616,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 31042572288,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 31336173568,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.27.ffn_up.weight",
"offset": 31629774848,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 31923376128,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 31923404800,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.27.attn_output.weight",
"offset": 31938084864,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.27.attn_q.weight",
"offset": 32040845312,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.27.attn_v.weight",
"offset": 32143605760,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.28.attn_norm.weight",
"offset": 32158285824,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 32158314496,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 32451915776,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.28.ffn_up.weight",
"offset": 32745517056,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 33039118336,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 33039147008,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.28.attn_output.weight",
"offset": 33053827072,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.28.attn_q.weight",
"offset": 33156587520,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.28.attn_v.weight",
"offset": 33259347968,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 33274028032,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.29.attn_k.weight",
"offset": 33567629312,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.29.attn_output.weight",
"offset": 33582309376,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.29.attn_q.weight",
"offset": 33685069824,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.29.attn_v.weight",
"offset": 33787830272,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.29.attn_norm.weight",
"offset": 33802510336,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 33802539008,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.29.ffn_up.weight",
"offset": 34096140288,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 34389741568,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.30.attn_norm.weight",
"offset": 34389770240,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 34389798912,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 34683400192,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.30.ffn_up.weight",
"offset": 34977001472,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 35270602752,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 35270631424,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.30.attn_output.weight",
"offset": 35285311488,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.30.attn_q.weight",
"offset": 35388071936,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.30.attn_v.weight",
"offset": 35490832384,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.31.attn_norm.weight",
"offset": 35505512448,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 35505541120,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 35799142400,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.31.ffn_up.weight",
"offset": 36092743680,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 36386344960,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 36386373632,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.31.attn_output.weight",
"offset": 36401053696,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.31.attn_q.weight",
"offset": 36503814144,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.31.attn_v.weight",
"offset": 36606574592,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.32.attn_norm.weight",
"offset": 36621254656,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 36621283328,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 36914884608,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.32.ffn_up.weight",
"offset": 37208485888,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 37502087168,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 37502115840,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.32.attn_output.weight",
"offset": 37516795904,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.32.attn_q.weight",
"offset": 37619556352,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.32.attn_v.weight",
"offset": 37722316800,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 37736996864,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.33.ffn_up.weight",
"offset": 38030598144,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.33.attn_k.weight",
"offset": 38324199424,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.33.attn_output.weight",
"offset": 38338879488,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.33.attn_q.weight",
"offset": 38441639936,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.33.attn_v.weight",
"offset": 38544400384,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.33.attn_norm.weight",
"offset": 38559080448,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 38559109120,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 38852710400,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.34.attn_norm.weight",
"offset": 38852739072,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 38852767744,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 39146369024,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.34.ffn_up.weight",
"offset": 39439970304,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 39733571584,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 39733600256,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.34.attn_output.weight",
"offset": 39748280320,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.34.attn_q.weight",
"offset": 39851040768,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.34.attn_v.weight",
"offset": 39953801216,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.35.attn_norm.weight",
"offset": 39968481280,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 39968509952,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 40262111232,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.35.ffn_up.weight",
"offset": 40555712512,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 40849313792,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 40849342464,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.35.attn_output.weight",
"offset": 40864022528,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.35.attn_q.weight",
"offset": 40966782976,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.35.attn_v.weight",
"offset": 41069543424,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.36.attn_norm.weight",
"offset": 41084223488,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 41084252160,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 41377853440,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.36.ffn_up.weight",
"offset": 41671454720,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 41965056000,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 41965084672,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.36.attn_output.weight",
"offset": 41979764736,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.36.attn_q.weight",
"offset": 42082525184,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.36.attn_v.weight",
"offset": 42185285632,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.37.attn_norm.weight",
"offset": 42199965696,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 42199994368,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 42493595648,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.37.ffn_up.weight",
"offset": 42787196928,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 43080798208,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 43080826880,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.37.attn_output.weight",
"offset": 43095506944,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.37.attn_q.weight",
"offset": 43198267392,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.37.attn_v.weight",
"offset": 43301027840,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.38.attn_k.weight",
"offset": 43315707904,
"shape": [
7168,
1024
],
"size": 14680064,
"type": 1
},
{
"name": "blk.38.attn_output.weight",
"offset": 43330387968,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.38.attn_q.weight",
"offset": 43433148416,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.38.attn_v.weight",
"offset": 43535908864,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.38.attn_norm.weight",
"offset": 43550588928,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 43550617600,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 43844218880,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.38.ffn_up.weight",
"offset": 44137820160,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 44431421440,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 44431450112,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 44431478784,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 44725080064,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.39.ffn_up.weight",
"offset": 45018681344,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 45312282624,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 45312311296,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.39.attn_output.weight",
"offset": 45326991360,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.39.attn_q.weight",
"offset": 45429751808,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.39.attn_v.weight",
"offset": 45532512256,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.40.attn_norm.weight",
"offset": 45547192320,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": " \ufffd\ufffd\u000e\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.55.",
"offset": 45547220992,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 45840822272,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.40.ffn_up.weight",
"offset": 46134423552,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 46428024832,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 46428053504,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.40.attn_output.weight",
"offset": 46442733568,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.40.attn_q.weight",
"offset": 46545494016,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.40.attn_v.weight",
"offset": 46648254464,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.41.attn_norm.weight",
"offset": 46662934528,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 46662963200,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 46956564480,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.41.ffn_up.weight",
"offset": 47250165760,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 47543767040,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 47543795712,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.41.attn_output.weight",
"offset": 47558475776,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.41.attn_q.weight",
"offset": 47661236224,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.41.attn_v.weight",
"offset": 47763996672,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 47778676736,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.42.attn_k.weight",
"offset": 48072278016,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.42.attn_output.weight",
"offset": 48086958080,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.42.attn_q.weight",
"offset": 48189718528,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.42.attn_v.weight",
"offset": 48292478976,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.42.attn_norm.weight",
"offset": 48307159040,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 48307187712,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.42.ffn_up.weight",
"offset": 48600788992,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 48894390272,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.43.attn_norm.weight",
"offset": 48894418944,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 48894447616,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 49188048896,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.43.ffn_up.weight",
"offset": 49481650176,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 49775251456,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 49775280128,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.43.attn_output.weight",
"offset": 49789960192,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.43.attn_q.weight",
"offset": 49892720640,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.43.attn_v.weight",
"offset": 49995481088,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.44.attn_norm.weight",
"offset": 50010161152,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 50010189824,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 50303791104,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.44.ffn_up.weight",
"offset": 50597392384,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 50890993664,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 50891022336,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.44.attn_output.weight",
"offset": 50905702400,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.44.attn_q.weight",
"offset": 51008462848,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.44.attn_v.weight",
"offset": 51111223296,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.45.attn_norm.weight",
"offset": 51125903360,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 51125932032,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 51419533312,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.45.ffn_up.weight",
"offset": 51713134592,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 52006735872,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 52006764544,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.45.attn_output.weight",
"offset": 52021444608,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.45.attn_q.weight",
"offset": 52124205056,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.45.attn_v.weight",
"offset": 52226965504,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 52241645568,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.46.ffn_up.weight",
"offset": 52535246848,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.46.attn_k.weight",
"offset": 52828848128,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.46.attn_output.weight",
"offset": 52843528192,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.46.attn_q.weight",
"offset": 52946288640,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.46.attn_v.weight",
"offset": 53049049088,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.46.attn_norm.weight",
"offset": 53063729152,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 53063757824,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 53357359104,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.47.attn_norm.weight",
"offset": 53357387776,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 53357416448,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 53651017728,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.47.ffn_up.weight",
"offset": 53944619008,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 54238220288,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 54238248960,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.47.attn_output.weight",
"offset": 54252929024,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.47.attn_q.weight",
"offset": 54355689472,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.47.attn_v.weight",
"offset": 54458449920,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.48.attn_norm.weight",
"offset": 54473129984,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 54473158656,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 54766759936,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.48.ffn_up.weight",
"offset": 55060361216,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 55353962496,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 55353991168,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.48.attn_output.weight",
"offset": 55368671232,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.48.attn_q.weight",
"offset": 55471431680,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.48.attn_v.weight",
"offset": 55574192128,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.49.attn_norm.weight",
"offset": 55588872192,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 55588900864,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 55882502144,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.49.ffn_up.weight",
"offset": 56176103424,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 56469704704,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 56469733376,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.49.attn_output.weight",
"offset": 56484413440,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.49.attn_q.weight",
"offset": 56587173888,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.49.attn_v.weight",
"offset": 56689934336,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.50.attn_norm.weight",
"offset": 56704614400,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 56704643072,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 56998244352,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.50.ffn_up.weight",
"offset": 57291845632,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 57585446912,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 57585475584,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.50.attn_output.weight",
"offset": 57600155648,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.50.attn_q.weight",
"offset": 57702916096,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.50.attn_v.weight",
"offset": 57805676544,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.51.attn_k.weight",
"offset": 57820356608,
"shape": [
7168,
1024
],
"size": 14680064,
"type": 1
},
{
"name": "blk.51.attn_output.weight",
"offset": 57835036672,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.51.attn_q.weight",
"offset": 57937797120,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.51.attn_v.weight",
"offset": 58040557568,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.51.attn_norm.weight",
"offset": 58055237632,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 58055266304,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 58348867584,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.51.ffn_up.weight",
"offset": 58642468864,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 58936070144,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.52.attn_norm.weight",
"offset": 58936098816,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 58936127488,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 59229728768,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.52.ffn_up.weight",
"offset": 59523330048,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 59816931328,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 59816960000,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.52.attn_output.weight",
"offset": 59831640064,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.52.attn_q.weight",
"offset": 59934400512,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.52.attn_v.weight",
"offset": 60037160960,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.53.attn_norm.weight",
"offset": 60051841024,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 60051869696,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 60345470976,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.53.ffn_up.weight",
"offset": 60639072256,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 60932673536,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 60932702208,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.53.attn_output.weight",
"offset": 60947382272,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.53.attn_q.weight",
"offset": 61050142720,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.53.attn_v.weight",
"offset": 61152903168,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.54.attn_norm.weight",
"offset": 61167583232,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 61167611904,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 61461213184,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.54.ffn_up.weight",
"offset": 61754814464,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 62048415744,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 62048444416,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.54.attn_output.weight",
"offset": 62063124480,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.54.attn_q.weight",
"offset": 62165884928,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.54.attn_v.weight",
"offset": 62268645376,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 62283325440,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.55.attn_k.weight",
"offset": 62576926720,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.55.attn_output.weight",
"offset": 62591606784,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.55.attn_q.weight",
"offset": 62694367232,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.55.attn_v.weight",
"offset": 62797127680,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.55.attn_norm.weight",
"offset": 62811807744,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 62811836416,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.55.ffn_up.weight",
"offset": 63105437696,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 63399038976,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.56.attn_norm.weight",
"offset": 63399067648,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 63399096320,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 63692697600,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.56.ffn_up.weight",
"offset": 63986298880,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 64279900160,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 64279928832,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.56.attn_output.weight",
"offset": 64294608896,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.56.attn_q.weight",
"offset": 64397369344,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.56.attn_v.weight",
"offset": 64500129792,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.57.attn_norm.weight",
"offset": 64514809856,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 64514838528,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 64808439808,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.57.ffn_up.weight",
"offset": 65102041088,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 65395642368,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 65395671040,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.57.attn_output.weight",
"offset": 65410351104,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.57.attn_q.weight",
"offset": 65513111552,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.57.attn_v.weight",
"offset": 65615872000,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.58.attn_norm.weight",
"offset": 65630552064,
"shape": [
7168
],
"size": 14680064,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 65630580736,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 65924182016,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.58.ffn_up.weight",
"offset": 66217783296,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 66511384576,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 66511413248,
"shape": [
7168,
1024
],
"size": 28672,
"type": 1
},
{
"name": "blk.58.attn_output.weight",
"offset": 66526093312,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.58.attn_q.weight",
"offset": 66628853760,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.58.attn_v.weight",
"offset": 66731614208,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 66746294272,
"shape": [
7168,
20480
],
"size": 14680064,
"type": 1
},
{
"name": "blk.59.ffn_up.weight",
"offset": 67039895552,
"shape": [
7168,
20480
],
"size": 293601280,
"type": 1
},
{
"name": "blk.59.attn_k.weight",
"offset": 67333496832,
"shape": [
7168,
1024
],
"size": 293601280,
"type": 1
},
{
"name": "blk.59.attn_output.weight",
"offset": 67348176896,
"shape": [
7168,
7168
],
"size": 14680064,
"type": 1
},
{
"name": "blk.59.attn_q.weight",
"offset": 67450937344,
"shape": [
7168,
7168
],
"size": 102760448,
"type": 1
},
{
"name": "blk.59.attn_v.weight",
"offset": 67553697792,
"shape": [
7168,
1024
],
"size": 102760448,
"type": 1
},
{
"name": "output.weight",
"offset": 67568377856,
"shape": [
7168,
64000
],
"size": 14680064,
"type": 1
},
{
"name": "blk.59.attn_norm.weight",
"offset": 68485881856,
"shape": [
7168
],
"size": 917504000,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 68485910528,
"shape": [
20480,
7168
],
"size": 28672,
"type": 1
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 68779511808,
"shape": [
7168
],
"size": 293601280,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 68779540480,
"shape": [
7168
],
"size": 28672,
"type": 0
}
],
"version": 3
}