Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
53dee60111ae · 13GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 10,
"general.name": "Yi-1.5-34B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 56,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 60,
"llama.context_length": 16384,
"llama.embedding_length": 7168,
"llama.feed_forward_length": 20480,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 34388917248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 150528000,
"shape": [
7168
],
"size": 150528000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 150556672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 213635072,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.0.ffn_up.weight",
"offset": 261804032,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 309972992,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 310001664,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.0.attn_output.weight",
"offset": 312410112,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.0.attn_q.weight",
"offset": 334487552,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.0.attn_v.weight",
"offset": 351346688,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.1.attn_norm.weight",
"offset": 355475456,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 355504128,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 418582528,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.1.ffn_up.weight",
"offset": 466751488,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 514920448,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 514949120,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.1.attn_output.weight",
"offset": 517357568,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.1.attn_q.weight",
"offset": 539435008,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.1.attn_v.weight",
"offset": 556294144,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.2.attn_norm.weight",
"offset": 560422912,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 560451584,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 623529984,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.2.ffn_up.weight",
"offset": 671698944,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 719867904,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 719896576,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.2.attn_output.weight",
"offset": 722305024,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.2.attn_q.weight",
"offset": 744382464,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.2.attn_v.weight",
"offset": 761241600,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 765370368,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.3.attn_k.weight",
"offset": 813539328,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.3.attn_output.weight",
"offset": 815947776,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.3.attn_q.weight",
"offset": 838025216,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.3.attn_v.weight",
"offset": 854884352,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.3.attn_norm.weight",
"offset": 859013120,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 859041792,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.3.ffn_up.weight",
"offset": 922120192,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 970289152,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.4.attn_norm.weight",
"offset": 970317824,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 970346496,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1033424896,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1081593856,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1129762816,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 1129791488,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.4.attn_output.weight",
"offset": 1132199936,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.4.attn_q.weight",
"offset": 1154277376,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.4.attn_v.weight",
"offset": 1171136512,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.5.attn_norm.weight",
"offset": 1175265280,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 1175293952,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1238372352,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.5.ffn_up.weight",
"offset": 1286541312,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 1334710272,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 1334738944,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.5.attn_output.weight",
"offset": 1337147392,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.5.attn_q.weight",
"offset": 1359224832,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.5.attn_v.weight",
"offset": 1376083968,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 1380212736,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 1380241408,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 1443319808,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.6.ffn_up.weight",
"offset": 1491488768,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 1539657728,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 1539686400,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.6.attn_output.weight",
"offset": 1542094848,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.6.attn_q.weight",
"offset": 1564172288,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.6.attn_v.weight",
"offset": 1581031424,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 1585160192,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.7.ffn_up.weight",
"offset": 1633329152,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.7.attn_k.weight",
"offset": 1681498112,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.7.attn_output.weight",
"offset": 1683906560,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.7.attn_q.weight",
"offset": 1705984000,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.7.attn_v.weight",
"offset": 1722843136,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.10.attn_norm.weight",
"offset": 1726971904,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 1727000576,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 1790078976,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.10.ffn_up.weight",
"offset": 1838247936,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 1886416896,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 1886445568,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.10.attn_output.weight",
"offset": 1888854016,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.10.attn_q.weight",
"offset": 1910931456,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.10.attn_v.weight",
"offset": 1927790592,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 1931919360,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 1931948032,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 1995026432,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.11.ffn_up.weight",
"offset": 2043195392,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 2091364352,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 2091393024,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.11.attn_output.weight",
"offset": 2093801472,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.11.attn_q.weight",
"offset": 2115878912,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.11.attn_v.weight",
"offset": 2132738048,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.12.attn_k.weight",
"offset": 2136866816,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 10
},
{
"name": "blk.12.attn_output.weight",
"offset": 2139275264,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.12.attn_q.weight",
"offset": 2161352704,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.12.attn_v.weight",
"offset": 2178211840,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2182340608,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2182369280,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 2245447680,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 2245476352,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 2245505024,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 2308583424,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.8.ffn_up.weight",
"offset": 2356752384,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 2404921344,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 2404950016,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.8.attn_output.weight",
"offset": 2407358464,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.8.attn_q.weight",
"offset": 2429435904,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.8.attn_v.weight",
"offset": 2446295040,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 2450423808,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 2450452480,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 2513530880,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.9.ffn_up.weight",
"offset": 2561699840,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 2609868800,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 2609897472,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.9.attn_output.weight",
"offset": 2612305920,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.9.attn_q.weight",
"offset": 2634383360,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.9.attn_v.weight",
"offset": 2651242496,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 2655371264,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 2655399936,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 2718478336,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.12.ffn_up.weight",
"offset": 2766647296,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 2814816256,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 2814844928,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 2814873600,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 2877952000,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.13.ffn_up.weight",
"offset": 2926120960,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 2974289920,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 2974318592,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.13.attn_output.weight",
"offset": 2976727040,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.13.attn_q.weight",
"offset": 2998804480,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.13.attn_v.weight",
"offset": 3015663616,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 3019792384,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 3019821056,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 3082899456,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.14.ffn_up.weight",
"offset": 3131068416,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 3179237376,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 3179266048,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.14.attn_output.weight",
"offset": 3181674496,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.14.attn_q.weight",
"offset": 3203751936,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.14.attn_v.weight",
"offset": 3220611072,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 3224739840,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 3224768512,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 3287846912,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.15.ffn_up.weight",
"offset": 3336015872,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 3384184832,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 3384213504,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.15.attn_output.weight",
"offset": 3386621952,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.15.attn_q.weight",
"offset": 3408699392,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.15.attn_v.weight",
"offset": 3425558528,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 3429687296,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.16.attn_k.weight",
"offset": 3477856256,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.16.attn_output.weight",
"offset": 3480264704,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.16.attn_q.weight",
"offset": 3502342144,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.16.attn_v.weight",
"offset": 3519201280,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 3523330048,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 3523358720,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.16.ffn_up.weight",
"offset": 3586437120,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 3634606080,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.17.attn_norm.weight",
"offset": 3634634752,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 3634663424,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 3697741824,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.17.ffn_up.weight",
"offset": 3745910784,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 3794079744,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 3794108416,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.17.attn_output.weight",
"offset": 3796516864,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.17.attn_q.weight",
"offset": 3818594304,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.17.attn_v.weight",
"offset": 3835453440,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 3839582208,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 3839610880,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 3902689280,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.18.ffn_up.weight",
"offset": 3950858240,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 3999027200,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 3999055872,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.18.attn_output.weight",
"offset": 4001464320,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.18.attn_q.weight",
"offset": 4023541760,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.18.attn_v.weight",
"offset": 4040400896,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 4044529664,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 4044558336,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 4107636736,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.19.ffn_up.weight",
"offset": 4155805696,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 4203974656,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 4204003328,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.19.attn_output.weight",
"offset": 4206411776,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.19.attn_q.weight",
"offset": 4228489216,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.19.attn_v.weight",
"offset": 4245348352,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 4249477120,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.20.ffn_up.weight",
"offset": 4297646080,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.20.attn_k.weight",
"offset": 4345815040,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.20.attn_output.weight",
"offset": 4348223488,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.20.attn_q.weight",
"offset": 4370300928,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.20.attn_v.weight",
"offset": 4387160064,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 4391288832,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 4391317504,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 4454395904,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 4454424576,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 4454453248,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 4517531648,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.21.ffn_up.weight",
"offset": 4565700608,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 4613869568,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 4613898240,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.21.attn_output.weight",
"offset": 4616306688,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.21.attn_q.weight",
"offset": 4638384128,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.21.attn_v.weight",
"offset": 4655243264,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 4659372032,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 4659400704,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 4722479104,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.22.ffn_up.weight",
"offset": 4770648064,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 4818817024,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 4818845696,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.22.attn_output.weight",
"offset": 4821254144,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.22.attn_q.weight",
"offset": 4843331584,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.22.attn_v.weight",
"offset": 4860190720,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 4864319488,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 4864348160,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 4927426560,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.23.ffn_up.weight",
"offset": 4975595520,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 5023764480,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 5023793152,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.23.attn_output.weight",
"offset": 5026201600,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.23.attn_q.weight",
"offset": 5048279040,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.23.attn_v.weight",
"offset": 5065138176,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.24.attn_norm.weight",
"offset": 5069266944,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 5069295616,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 5132374016,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.24.ffn_up.weight",
"offset": 5180542976,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 5228711936,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 5228740608,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.24.attn_output.weight",
"offset": 5231149056,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.24.attn_q.weight",
"offset": 5253226496,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.24.attn_v.weight",
"offset": 5270085632,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.25.attn_k.weight",
"offset": 5274214400,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 10
},
{
"name": "blk.25.attn_output.weight",
"offset": 5276622848,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.25.attn_q.weight",
"offset": 5298700288,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.25.attn_v.weight",
"offset": 5315559424,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 5319688192,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 5319716864,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 5382795264,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.25.ffn_up.weight",
"offset": 5430964224,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 5479133184,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.26.attn_norm.weight",
"offset": 5479161856,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 5479190528,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 5542268928,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.26.ffn_up.weight",
"offset": 5590437888,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 5638606848,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 5638635520,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.26.attn_output.weight",
"offset": 5641043968,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.26.attn_q.weight",
"offset": 5663121408,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.26.attn_v.weight",
"offset": 5679980544,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.27.attn_norm.weight",
"offset": 5684109312,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 5684137984,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 5747216384,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.27.ffn_up.weight",
"offset": 5795385344,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 5843554304,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 5843582976,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.27.attn_output.weight",
"offset": 5845991424,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.27.attn_q.weight",
"offset": 5868068864,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.27.attn_v.weight",
"offset": 5884928000,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 5889056768,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 5889085440,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 5952163840,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.28.ffn_up.weight",
"offset": 6000332800,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 6048501760,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 6048530432,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.28.attn_output.weight",
"offset": 6050938880,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.28.attn_q.weight",
"offset": 6073016320,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.28.attn_v.weight",
"offset": 6089875456,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 6094004224,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.29.attn_k.weight",
"offset": 6142173184,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.29.attn_output.weight",
"offset": 6144581632,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.29.attn_q.weight",
"offset": 6166659072,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.29.attn_v.weight",
"offset": 6183518208,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 6187646976,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 6187675648,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.29.ffn_up.weight",
"offset": 6250754048,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 6298923008,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.30.attn_norm.weight",
"offset": 6298951680,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 6298980352,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 6362058752,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.30.ffn_up.weight",
"offset": 6410227712,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 6458396672,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 6458425344,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.30.attn_output.weight",
"offset": 6460833792,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.30.attn_q.weight",
"offset": 6482911232,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.30.attn_v.weight",
"offset": 6499770368,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.31.attn_norm.weight",
"offset": 6503899136,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 6503927808,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 6567006208,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.31.ffn_up.weight",
"offset": 6615175168,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 6663344128,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 6663372800,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.31.attn_output.weight",
"offset": 6665781248,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.31.attn_q.weight",
"offset": 6687858688,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.31.attn_v.weight",
"offset": 6704717824,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.32.attn_norm.weight",
"offset": 6708846592,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 6708875264,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 6771953664,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.32.ffn_up.weight",
"offset": 6820122624,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 6868291584,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 6868320256,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.32.attn_output.weight",
"offset": 6870728704,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.32.attn_q.weight",
"offset": 6892806144,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.32.attn_v.weight",
"offset": 6909665280,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 6913794048,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.33.ffn_up.weight",
"offset": 6961963008,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.33.attn_k.weight",
"offset": 7010131968,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.33.attn_output.weight",
"offset": 7012540416,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.33.attn_q.weight",
"offset": 7034617856,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.33.attn_v.weight",
"offset": 7051476992,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.33.attn_norm.weight",
"offset": 7055605760,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 7055634432,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 7118712832,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.34.attn_norm.weight",
"offset": 7118741504,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 7118770176,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 7181848576,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.34.ffn_up.weight",
"offset": 7230017536,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 7278186496,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 7278215168,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.34.attn_output.weight",
"offset": 7280623616,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.34.attn_q.weight",
"offset": 7302701056,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.34.attn_v.weight",
"offset": 7319560192,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.35.attn_norm.weight",
"offset": 7323688960,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 7323717632,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 7386796032,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.35.ffn_up.weight",
"offset": 7434964992,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 7483133952,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 7483162624,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.35.attn_output.weight",
"offset": 7485571072,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.35.attn_q.weight",
"offset": 7507648512,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.35.attn_v.weight",
"offset": 7524507648,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.36.attn_norm.weight",
"offset": 7528636416,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 7528665088,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 7591743488,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.36.ffn_up.weight",
"offset": 7639912448,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 7688081408,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 7688110080,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.36.attn_output.weight",
"offset": 7690518528,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.36.attn_q.weight",
"offset": 7712595968,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.36.attn_v.weight",
"offset": 7729455104,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.37.attn_norm.weight",
"offset": 7733583872,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 7733612544,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 7796690944,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.37.ffn_up.weight",
"offset": 7844859904,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 7893028864,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 7893057536,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.37.attn_output.weight",
"offset": 7895465984,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.37.attn_q.weight",
"offset": 7917543424,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.37.attn_v.weight",
"offset": 7934402560,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.38.attn_k.weight",
"offset": 7938531328,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 10
},
{
"name": "blk.38.attn_output.weight",
"offset": 7940939776,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.38.attn_q.weight",
"offset": 7963017216,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.38.attn_v.weight",
"offset": 7979876352,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.38.attn_norm.weight",
"offset": 7984005120,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 7984033792,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 8047112192,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.38.ffn_up.weight",
"offset": 8095281152,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 8143450112,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 8143478784,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 8143507456,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 8206585856,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.39.ffn_up.weight",
"offset": 8254754816,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 8302923776,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 8302952448,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.39.attn_output.weight",
"offset": 8305360896,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.39.attn_q.weight",
"offset": 8327438336,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.39.attn_v.weight",
"offset": 8344297472,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.40.attn_norm.weight",
"offset": 8348426240,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 8348454912,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 8411533312,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.40.ffn_up.weight",
"offset": 8459702272,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 8507871232,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 8507899904,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.40.attn_output.weight",
"offset": 8510308352,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.40.attn_q.weight",
"offset": 8532385792,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.40.attn_v.weight",
"offset": 8549244928,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.41.attn_norm.weight",
"offset": 8553373696,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 8553402368,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 8616480768,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.41.ffn_up.weight",
"offset": 8664649728,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 8712818688,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 8712847360,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.41.attn_output.weight",
"offset": 8715255808,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.41.attn_q.weight",
"offset": 8737333248,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.41.attn_v.weight",
"offset": 8754192384,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 8758321152,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.42.attn_k.weight",
"offset": 8806490112,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.42.attn_output.weight",
"offset": 8808898560,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.42.attn_q.weight",
"offset": 8830976000,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.42.attn_v.weight",
"offset": 8847835136,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.42.attn_norm.weight",
"offset": 8851963904,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 8851992576,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.42.ffn_up.weight",
"offset": 8915070976,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 8963239936,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.43.attn_norm.weight",
"offset": 8963268608,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 8963297280,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 9026375680,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.43.ffn_up.weight",
"offset": 9074544640,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 9122713600,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 9122742272,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.43.attn_output.weight",
"offset": 9125150720,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.43.attn_q.weight",
"offset": 9147228160,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.43.attn_v.weight",
"offset": 9164087296,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.44.attn_norm.weight",
"offset": 9168216064,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 9168244736,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 9231323136,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.44.ffn_up.weight",
"offset": 9279492096,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 9327661056,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 9327689728,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.44.attn_output.weight",
"offset": 9330098176,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.44.attn_q.weight",
"offset": 9352175616,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.44.attn_v.weight",
"offset": 9369034752,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.45.attn_norm.weight",
"offset": 9373163520,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 9373192192,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 9436270592,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.45.ffn_up.weight",
"offset": 9484439552,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 9532608512,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 9532637184,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.45.attn_output.weight",
"offset": 9535045632,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.45.attn_q.weight",
"offset": 9557123072,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.45.attn_v.weight",
"offset": 9573982208,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 9578110976,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.46.ffn_up.weight",
"offset": 9626279936,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.46.attn_k.weight",
"offset": 9674448896,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.46.attn_output.weight",
"offset": 9676857344,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.46.attn_q.weight",
"offset": 9698934784,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.46.attn_v.weight",
"offset": 9715793920,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.46.attn_norm.weight",
"offset": 9719922688,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 9719951360,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 9783029760,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "blk.47.attn_norm.weight",
"offset": 9783058432,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 9783087104,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 9846165504,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.47.ffn_up.weight",
"offset": 9894334464,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 9942503424,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 9942532096,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "\u0000\u001c\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u001c\u0000\u0000\u0000\u0000\u0000\u0000\u000b\u0000\u0000\u0000\u0000\ufffd\ufffdP\u0002",
"offset": 9944940544,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.47.attn_q.weight",
"offset": 9967017984,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.47.attn_v.weight",
"offset": 9983877120,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.48.attn_norm.weight",
"offset": 9988005888,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 9988034560,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 10051112960,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.48.ffn_up.weight",
"offset": 10099281920,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 10147450880,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 10147479552,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.48.attn_output.weight",
"offset": 10149888000,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.48.attn_q.weight",
"offset": 10171965440,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.48.attn_v.weight",
"offset": 10188824576,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.49.attn_norm.weight",
"offset": 10192953344,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 10192982016,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 10256060416,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.49.ffn_up.weight",
"offset": 10304229376,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 10352398336,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 10352427008,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.49.attn_output.weight",
"offset": 10354835456,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.49.attn_q.weight",
"offset": 10376912896,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.49.attn_v.weight",
"offset": 10393772032,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.50.attn_norm.weight",
"offset": 10397900800,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 10397929472,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 10461007872,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.50.ffn_up.weight",
"offset": 10509176832,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 10557345792,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 10557374464,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.50.attn_output.weight",
"offset": 10559782912,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.50.attn_q.weight",
"offset": 10581860352,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.50.attn_v.weight",
"offset": 10598719488,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.51.attn_k.weight",
"offset": 10602848256,
"shape": [
7168,
1024
],
"size": 4128768,
"type": 10
},
{
"name": "blk.51.attn_output.weight",
"offset": 10605256704,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.51.attn_q.weight",
"offset": 10627334144,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.51.attn_v.weight",
"offset": 10644193280,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.51.attn_norm.weight",
"offset": 10648322048,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 10648350720,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 10711429120,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.51.ffn_up.weight",
"offset": 10759598080,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 10807767040,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.52.attn_norm.weight",
"offset": 10807795712,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 10807824384,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 10870902784,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.52.ffn_up.weight",
"offset": 10919071744,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 10967240704,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 10967269376,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.52.attn_output.weight",
"offset": 10969677824,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.52.attn_q.weight",
"offset": 10991755264,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.52.attn_v.weight",
"offset": 11008614400,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.53.attn_norm.weight",
"offset": 11012743168,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 11012771840,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 11075850240,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.53.ffn_up.weight",
"offset": 11124019200,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 11172188160,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 11172216832,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.53.attn_output.weight",
"offset": 11174625280,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.53.attn_q.weight",
"offset": 11196702720,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.53.attn_v.weight",
"offset": 11213561856,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.54.attn_norm.weight",
"offset": 11217690624,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 11217719296,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 11280797696,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.54.ffn_up.weight",
"offset": 11328966656,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 11377135616,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 11377164288,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.54.attn_output.weight",
"offset": 11379572736,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.54.attn_q.weight",
"offset": 11401650176,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.54.attn_v.weight",
"offset": 11418509312,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 11422638080,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.55.attn_k.weight",
"offset": 11470807040,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.55.attn_output.weight",
"offset": 11473215488,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.55.attn_q.weight",
"offset": 11495292928,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.55.attn_v.weight",
"offset": 11512152064,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.55.attn_norm.weight",
"offset": 11516280832,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 11516309504,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.55.ffn_up.weight",
"offset": 11579387904,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 11627556864,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.56.attn_norm.weight",
"offset": 11627585536,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 11627614208,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 11690692608,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.56.ffn_up.weight",
"offset": 11738861568,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 11787030528,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 11787059200,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.56.attn_output.weight",
"offset": 11789467648,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.56.attn_q.weight",
"offset": 11811545088,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.56.attn_v.weight",
"offset": 11828404224,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.57.attn_norm.weight",
"offset": 11832532992,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 11832561664,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 11895640064,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.57.ffn_up.weight",
"offset": 11943809024,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 11991977984,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 11992006656,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.57.attn_output.weight",
"offset": 11994415104,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.57.attn_q.weight",
"offset": 12016492544,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.57.attn_v.weight",
"offset": 12033351680,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.58.attn_norm.weight",
"offset": 12037480448,
"shape": [
7168
],
"size": 4128768,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 12037509120,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 12100587520,
"shape": [
7168,
20480
],
"size": 63078400,
"type": 10
},
{
"name": "blk.58.ffn_up.weight",
"offset": 12148756480,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 12196925440,
"shape": [
7168
],
"size": 48168960,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 12196954112,
"shape": [
7168,
1024
],
"size": 28672,
"type": 10
},
{
"name": "blk.58.attn_output.weight",
"offset": 12199362560,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.58.attn_q.weight",
"offset": 12221440000,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.58.attn_v.weight",
"offset": 12238299136,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 12242427904,
"shape": [
7168,
20480
],
"size": 4128768,
"type": 10
},
{
"name": "blk.59.ffn_up.weight",
"offset": 12290596864,
"shape": [
7168,
20480
],
"size": 48168960,
"type": 10
},
{
"name": "blk.59.attn_k.weight",
"offset": 12338765824,
"shape": [
7168,
1024
],
"size": 48168960,
"type": 10
},
{
"name": "blk.59.attn_output.weight",
"offset": 12341174272,
"shape": [
7168,
7168
],
"size": 2408448,
"type": 11
},
{
"name": "blk.59.attn_q.weight",
"offset": 12363251712,
"shape": [
7168,
7168
],
"size": 22077440,
"type": 10
},
{
"name": "blk.59.attn_v.weight",
"offset": 12380110848,
"shape": [
7168,
1024
],
"size": 16859136,
"type": 12
},
{
"name": "output.weight",
"offset": 12384239616,
"shape": [
7168,
64000
],
"size": 4128768,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 12760559616,
"shape": [
7168
],
"size": 376320000,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 12760588288,
"shape": [
20480,
7168
],
"size": 28672,
"type": 11
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 12823666688,
"shape": [
7168
],
"size": 63078400,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 12823695360,
"shape": [
7168
],
"size": 28672,
"type": 0
}
],
"version": 3
}