Yi-1.5 is an upgraded version of Yi, 9B and 34B with context windows 16K here.
9B
34B
190 Pulls Updated 3 weeks ago
4c5cc9fb0df1 · 28GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 18,
"general.name": "Yi-1.5-34B-Chat-16K",
"general.quantization_version": 2,
"llama.attention.head_count": 56,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.000001,
"llama.block_count": 60,
"llama.context_length": 16384,
"llama.embedding_length": 7168,
"llama.feed_forward_length": 20480,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 5000000,
"llama.vocab_size": 64000,
"tokenizer.ggml.add_bos_token": false,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 7,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.padding_token_id": 0,
"tokenizer.ggml.scores": "... (64000 values)",
"tokenizer.ggml.token_type": "... (64000 values)",
"tokenizer.ggml.tokens": "... (64000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 34388917248,
"tensors": [
{
"name": "blk.0.attn_norm.weight",
"offset": 376320000,
"shape": [
7168
],
"size": 376320000,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 376348672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 496771072,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.0.ffn_up.weight",
"offset": 617193472,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 737615872,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 737644544,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 743665664,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.0.attn_q.weight",
"offset": 785813504,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.0.attn_v.weight",
"offset": 827961344,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 833982464,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 834011136,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 954433536,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1074855936,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1195278336,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1195307008,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 1201328128,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.1.attn_q.weight",
"offset": 1243475968,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.1.attn_v.weight",
"offset": 1285623808,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1291644928,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1291673600,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1412096000,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1532518400,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1652940800,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 1652969472,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 1658990592,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.2.attn_q.weight",
"offset": 1701138432,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.2.attn_v.weight",
"offset": 1743286272,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1749307392,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.3.attn_k.weight",
"offset": 1869729792,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 1875750912,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.3.attn_q.weight",
"offset": 1917898752,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.3.attn_v.weight",
"offset": 1960046592,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1966067712,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1966096384,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.3.ffn_up.weight",
"offset": 2086518784,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 2206941184,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.4.attn_norm.weight",
"offset": 2206969856,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 2206998528,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 2327420928,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.4.ffn_up.weight",
"offset": 2447843328,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 2568265728,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 2568294400,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 2574315520,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.4.attn_q.weight",
"offset": 2616463360,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.4.attn_v.weight",
"offset": 2658611200,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 2664632320,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 2664660992,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 2785083392,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.5.ffn_up.weight",
"offset": 2905505792,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 3025928192,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 3025956864,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 3031977984,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.5.attn_q.weight",
"offset": 3074125824,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.5.attn_v.weight",
"offset": 3116273664,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 3122294784,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 3122323456,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 3242745856,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.6.ffn_up.weight",
"offset": 3363168256,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 3483590656,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 3483619328,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 3489640448,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.6.attn_q.weight",
"offset": 3531788288,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.6.attn_v.weight",
"offset": 3573936128,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 3579957248,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.7.ffn_up.weight",
"offset": 3700379648,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.7.attn_k.weight",
"offset": 3820802048,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 3826823168,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.7.attn_q.weight",
"offset": 3868971008,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.7.attn_v.weight",
"offset": 3911118848,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 3917139968,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 3917168640,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 4037591040,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.10.ffn_up.weight",
"offset": 4158013440,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 4278435840,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 4278464512,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 4284485632,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.10.attn_q.weight",
"offset": 4326633472,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.10.attn_v.weight",
"offset": 4368781312,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 4374802432,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 4374831104,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 4495253504,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.11.ffn_up.weight",
"offset": 4615675904,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 4736098304,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 4736126976,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 4742148096,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.11.attn_q.weight",
"offset": 4784295936,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.11.attn_v.weight",
"offset": 4826443776,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.12.attn_k.weight",
"offset": 4832464896,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 4838486016,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.12.attn_q.weight",
"offset": 4880633856,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.12.attn_v.weight",
"offset": 4922781696,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 4928802816,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 4928831488,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 5049253888,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.8.attn_norm.weight",
"offset": 5049282560,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 5049311232,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 5169733632,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.8.ffn_up.weight",
"offset": 5290156032,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 5410578432,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 5410607104,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 5416628224,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.8.attn_q.weight",
"offset": 5458776064,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.8.attn_v.weight",
"offset": 5500923904,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 5506945024,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 5506973696,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 5627396096,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.9.ffn_up.weight",
"offset": 5747818496,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 5868240896,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 5868269568,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 5874290688,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.9.attn_q.weight",
"offset": 5916438528,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.9.attn_v.weight",
"offset": 5958586368,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 5964607488,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 5964636160,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 6085058560,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.12.ffn_up.weight",
"offset": 6205480960,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 6325903360,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.13.attn_norm.weight",
"offset": 6325932032,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 6325960704,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 6446383104,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.13.ffn_up.weight",
"offset": 6566805504,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 6687227904,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 6687256576,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 6693277696,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.13.attn_q.weight",
"offset": 6735425536,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.13.attn_v.weight",
"offset": 6777573376,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 6783594496,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 6783623168,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 6904045568,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.14.ffn_up.weight",
"offset": 7024467968,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 7144890368,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 7144919040,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 7150940160,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.14.attn_q.weight",
"offset": 7193088000,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.14.attn_v.weight",
"offset": 7235235840,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 7241256960,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 7241285632,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 7361708032,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.15.ffn_up.weight",
"offset": 7482130432,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 7602552832,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 7602581504,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 7608602624,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.15.attn_q.weight",
"offset": 7650750464,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.15.attn_v.weight",
"offset": 7692898304,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 7698919424,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.16.attn_k.weight",
"offset": 7819341824,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 7825362944,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.16.attn_q.weight",
"offset": 7867510784,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.16.attn_v.weight",
"offset": 7909658624,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 7915679744,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 7915708416,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.16.ffn_up.weight",
"offset": 8036130816,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 8156553216,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.17.attn_norm.weight",
"offset": 8156581888,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 8156610560,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 8277032960,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.17.ffn_up.weight",
"offset": 8397455360,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 8517877760,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 8517906432,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 8523927552,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.17.attn_q.weight",
"offset": 8566075392,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.17.attn_v.weight",
"offset": 8608223232,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 8614244352,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 8614273024,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 8734695424,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.18.ffn_up.weight",
"offset": 8855117824,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 8975540224,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 8975568896,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 8981590016,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.18.attn_q.weight",
"offset": 9023737856,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.18.attn_v.weight",
"offset": 9065885696,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 9071906816,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 9071935488,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 9192357888,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.19.ffn_up.weight",
"offset": 9312780288,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 9433202688,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 9433231360,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 9439252480,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.19.attn_q.weight",
"offset": 9481400320,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.19.attn_v.weight",
"offset": 9523548160,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 9529569280,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.20.ffn_up.weight",
"offset": 9649991680,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.20.attn_k.weight",
"offset": 9770414080,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 9776435200,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.20.attn_q.weight",
"offset": 9818583040,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.20.attn_v.weight",
"offset": 9860730880,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 9866752000,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 9866780672,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 9987203072,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.21.attn_norm.weight",
"offset": 9987231744,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 9987260416,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 10107682816,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.21.ffn_up.weight",
"offset": 10228105216,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 10348527616,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 10348556288,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 10354577408,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.21.attn_q.weight",
"offset": 10396725248,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.21.attn_v.weight",
"offset": 10438873088,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 10444894208,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 10444922880,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 10565345280,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.22.ffn_up.weight",
"offset": 10685767680,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 10806190080,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 10806218752,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 10812239872,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.22.attn_q.weight",
"offset": 10854387712,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.22.attn_v.weight",
"offset": 10896535552,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 10902556672,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 10902585344,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 11023007744,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.23.ffn_up.weight",
"offset": 11143430144,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 11263852544,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 11263881216,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 11269902336,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.23.attn_q.weight",
"offset": 11312050176,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.23.attn_v.weight",
"offset": 11354198016,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 11360219136,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 11360247808,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 11480670208,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.24.ffn_up.weight",
"offset": 11601092608,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 11721515008,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 11721543680,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 11727564800,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.24.attn_q.weight",
"offset": 11769712640,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.24.attn_v.weight",
"offset": 11811860480,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.25.attn_k.weight",
"offset": 11817881600,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 11823902720,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.25.attn_q.weight",
"offset": 11866050560,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.25.attn_v.weight",
"offset": 11908198400,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 11914219520,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 11914248192,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 12034670592,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.25.ffn_up.weight",
"offset": 12155092992,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 12275515392,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.26.attn_norm.weight",
"offset": 12275544064,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 12275572736,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 12395995136,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.26.ffn_up.weight",
"offset": 12516417536,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 12636839936,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 12636868608,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 12642889728,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.26.attn_q.weight",
"offset": 12685037568,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.26.attn_v.weight",
"offset": 12727185408,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 12733206528,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 12733235200,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 12853657600,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.27.ffn_up.weight",
"offset": 12974080000,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 13094502400,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 13094531072,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 13100552192,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.27.attn_q.weight",
"offset": 13142700032,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.27.attn_v.weight",
"offset": 13184847872,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.28.attn_norm.weight",
"offset": 13190868992,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 13190897664,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 13311320064,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.28.ffn_up.weight",
"offset": 13431742464,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 13552164864,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 13552193536,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.28.attn_output.weight",
"offset": 13558214656,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.28.attn_q.weight",
"offset": 13600362496,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.28.attn_v.weight",
"offset": 13642510336,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 13648531456,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.29.attn_k.weight",
"offset": 13768953856,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.29.attn_output.weight",
"offset": 13774974976,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.29.attn_q.weight",
"offset": 13817122816,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.29.attn_v.weight",
"offset": 13859270656,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 13865291776,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 13865320448,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.29.ffn_up.weight",
"offset": 13985742848,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 14106165248,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.30.attn_norm.weight",
"offset": 14106193920,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 14106222592,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 14226644992,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.30.ffn_up.weight",
"offset": 14347067392,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 14467489792,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 14467518464,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 14473539584,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.30.attn_q.weight",
"offset": 14515687424,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.30.attn_v.weight",
"offset": 14557835264,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 14563856384,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 14563885056,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 14684307456,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.31.ffn_up.weight",
"offset": 14804729856,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 14925152256,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 14925180928,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.31.attn_output.weight",
"offset": 14931202048,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.31.attn_q.weight",
"offset": 14973349888,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.31.attn_v.weight",
"offset": 15015497728,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.32.attn_norm.weight",
"offset": 15021518848,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 15021547520,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 15141969920,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.32.ffn_up.weight",
"offset": 15262392320,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 15382814720,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 15382843392,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.32.attn_output.weight",
"offset": 15388864512,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.32.attn_q.weight",
"offset": 15431012352,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.32.attn_v.weight",
"offset": 15473160192,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 15479181312,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.33.ffn_up.weight",
"offset": 15599603712,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.33.attn_k.weight",
"offset": 15720026112,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.33.attn_output.weight",
"offset": 15726047232,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.33.attn_q.weight",
"offset": 15768195072,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.33.attn_v.weight",
"offset": 15810342912,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 15816364032,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 15816392704,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 15936815104,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.34.attn_norm.weight",
"offset": 15936843776,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 15936872448,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 16057294848,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.34.ffn_up.weight",
"offset": 16177717248,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 16298139648,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 16298168320,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.34.attn_output.weight",
"offset": 16304189440,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.34.attn_q.weight",
"offset": 16346337280,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.34.attn_v.weight",
"offset": 16388485120,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.35.attn_norm.weight",
"offset": 16394506240,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 16394534912,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 16514957312,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.35.ffn_up.weight",
"offset": 16635379712,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 16755802112,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 16755830784,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.35.attn_output.weight",
"offset": 16761851904,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.35.attn_q.weight",
"offset": 16803999744,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.35.attn_v.weight",
"offset": 16846147584,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.36.attn_norm.weight",
"offset": 16852168704,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 16852197376,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 16972619776,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.36.ffn_up.weight",
"offset": 17093042176,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 17213464576,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 17213493248,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.36.attn_output.weight",
"offset": 17219514368,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.36.attn_q.weight",
"offset": 17261662208,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.36.attn_v.weight",
"offset": 17303810048,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 17309831168,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 17309859840,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 17430282240,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.37.ffn_up.weight",
"offset": 17550704640,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 17671127040,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 17671155712,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.37.attn_output.weight",
"offset": 17677176832,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.37.attn_q.weight",
"offset": 17719324672,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.37.attn_v.weight",
"offset": 17761472512,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.38.attn_k.weight",
"offset": 17767493632,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 14
},
{
"name": "blk.38.attn_output.weight",
"offset": 17773514752,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.38.attn_q.weight",
"offset": 17815662592,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.38.attn_v.weight",
"offset": 17857810432,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.38.attn_norm.weight",
"offset": 17863831552,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 17863860224,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 17984282624,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.38.ffn_up.weight",
"offset": 18104705024,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 18225127424,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.39.attn_norm.weight",
"offset": 18225156096,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 18225184768,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 18345607168,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.39.ffn_up.weight",
"offset": 18466029568,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 18586451968,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 18586480640,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.39.attn_output.weight",
"offset": 18592501760,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.39.attn_q.weight",
"offset": 18634649600,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.39.attn_v.weight",
"offset": 18676797440,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.40.attn_norm.weight",
"offset": 18682818560,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 18682847232,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 18803269632,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.40.ffn_up.weight",
"offset": 18923692032,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 19044114432,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 19044143104,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.40.attn_output.weight",
"offset": 19050164224,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.40.attn_q.weight",
"offset": 19092312064,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.40.attn_v.weight",
"offset": 19134459904,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.41.attn_norm.weight",
"offset": 19140481024,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 19140509696,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 19260932096,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.41.ffn_up.weight",
"offset": 19381354496,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 19501776896,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 19501805568,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.41.attn_output.weight",
"offset": 19507826688,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.41.attn_q.weight",
"offset": 19549974528,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.41.attn_v.weight",
"offset": 19592122368,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 19598143488,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.42.attn_k.weight",
"offset": 19718565888,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.42.attn_output.weight",
"offset": 19724587008,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.42.attn_q.weight",
"offset": 19766734848,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.42.attn_v.weight",
"offset": 19808882688,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 19814903808,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 19814932480,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.42.ffn_up.weight",
"offset": 19935354880,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 20055777280,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.43.attn_norm.weight",
"offset": 20055805952,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 20055834624,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 20176257024,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.43.ffn_up.weight",
"offset": 20296679424,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 20417101824,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 20417130496,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.43.attn_output.weight",
"offset": 20423151616,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.43.attn_q.weight",
"offset": 20465299456,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.43.attn_v.weight",
"offset": 20507447296,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.44.attn_norm.weight",
"offset": 20513468416,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 20513497088,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 20633919488,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.44.ffn_up.weight",
"offset": 20754341888,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 20874764288,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 20874792960,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.44.attn_output.weight",
"offset": 20880814080,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.44.attn_q.weight",
"offset": 20922961920,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.44.attn_v.weight",
"offset": 20965109760,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 20971130880,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 20971159552,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 21091581952,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.45.ffn_up.weight",
"offset": 21212004352,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 21332426752,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 21332455424,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.45.attn_output.weight",
"offset": 21338476544,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.45.attn_q.weight",
"offset": 21380624384,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.45.attn_v.weight",
"offset": 21422772224,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 21428793344,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.46.ffn_up.weight",
"offset": 21549215744,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.46.attn_k.weight",
"offset": 21669638144,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.46.attn_output.weight",
"offset": 21675659264,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.46.attn_q.weight",
"offset": 21717807104,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.46.attn_v.weight",
"offset": 21759954944,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.46.attn_norm.weight",
"offset": 21765976064,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 21766004736,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 21886427136,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.47.attn_norm.weight",
"offset": 21886455808,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 21886484480,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 22006906880,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.47.ffn_up.weight",
"offset": 22127329280,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 22247751680,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 22247780352,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.47.attn_output.weight",
"offset": 22253801472,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.47.attn_q.weight",
"offset": 22295949312,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.47.attn_v.weight",
"offset": 22338097152,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.48.attn_norm.weight",
"offset": 22344118272,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 22344146944,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 22464569344,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.48.ffn_up.weight",
"offset": 22584991744,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 22705414144,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 22705442816,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.48.attn_output.weight",
"offset": 22711463936,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.48.attn_q.weight",
"offset": 22753611776,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.48.attn_v.weight",
"offset": 22795759616,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.49.attn_norm.weight",
"offset": 22801780736,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 22801809408,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 22922231808,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.49.ffn_up.weight",
"offset": 23042654208,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 23163076608,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 23163105280,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.49.attn_output.weight",
"offset": 23169126400,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.49.attn_q.weight",
"offset": 23211274240,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.49.attn_v.weight",
"offset": 23253422080,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.50.attn_norm.weight",
"offset": 23259443200,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 23259471872,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 23379894272,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.50.ffn_up.weight",
"offset": 23500316672,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 23620739072,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 23620767744,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.50.attn_output.weight",
"offset": 23626788864,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.50.attn_q.weight",
"offset": 23668936704,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.50.attn_v.weight",
"offset": 23711084544,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.51.attn_k.weight",
"offset": 23717105664,
"shape": [
7168,
1024
],
"size": 6021120,
"type": 14
},
{
"name": "blk.51.attn_output.weight",
"offset": 23723126784,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.51.attn_q.weight",
"offset": 23765274624,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.51.attn_v.weight",
"offset": 23807422464,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.51.attn_norm.weight",
"offset": 23813443584,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 23813472256,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 23933894656,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.51.ffn_up.weight",
"offset": 24054317056,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 24174739456,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.52.attn_norm.weight",
"offset": 24174768128,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 24174796800,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 24295219200,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.52.ffn_up.weight",
"offset": 24415641600,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 24536064000,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 24536092672,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.52.attn_output.weight",
"offset": 24542113792,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.52.attn_q.weight",
"offset": 24584261632,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.52.attn_v.weight",
"offset": 24626409472,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.53.attn_norm.weight",
"offset": 24632430592,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 24632459264,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 24752881664,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.53.ffn_up.weight",
"offset": 24873304064,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 24993726464,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 24993755136,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.53.attn_output.weight",
"offset": 24999776256,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.53.attn_q.weight",
"offset": 25041924096,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.53.attn_v.weight",
"offset": 25084071936,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.54.attn_norm.weight",
"offset": 25090093056,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 25090121728,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 25210544128,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.54.ffn_up.weight",
"offset": 25330966528,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 25451388928,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 25451417600,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.54.attn_output.weight",
"offset": 25457438720,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.54.attn_q.weight",
"offset": 25499586560,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.54.attn_v.weight",
"offset": 25541734400,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 25547755520,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.55.attn_k.weight",
"offset": 25668177920,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.55.attn_output.weight",
"offset": 25674199040,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.55.attn_q.weight",
"offset": 25716346880,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.55.attn_v.weight",
"offset": 25758494720,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.55.attn_norm.weight",
"offset": 25764515840,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 25764544512,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.55.ffn_up.weight",
"offset": 25884966912,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 26005389312,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.56.attn_norm.weight",
"offset": 26005417984,
"shape": [
7168
],
"size": 28672,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 26005446656,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 26125869056,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.56.ffn_up.weight",
"offset": 26246291456,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 26366713856,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 26366742528,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.56.attn_output.weight",
"offset": 26372763648,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.56.attn_q.weight",
"offset": 26414911488,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.56.attn_v.weight",
"offset": 26457059328,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.57.attn_norm.weight",
"offset": 26463080448,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 26463109120,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 26583531520,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.57.ffn_up.weight",
"offset": 26703953920,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 26824376320,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 26824404992,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.57.attn_output.weight",
"offset": 26830426112,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.57.attn_q.weight",
"offset": 26872573952,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.57.attn_v.weight",
"offset": 26914721792,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.58.attn_norm.weight",
"offset": 26920742912,
"shape": [
7168
],
"size": 6021120,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 26920771584,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 27041193984,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.58.ffn_up.weight",
"offset": 27161616384,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 27282038784,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 27282067456,
"shape": [
7168,
1024
],
"size": 28672,
"type": 14
},
{
"name": "blk.58.attn_output.weight",
"offset": 27288088576,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.58.attn_q.weight",
"offset": 27330236416,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.58.attn_v.weight",
"offset": 27372384256,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 27378405376,
"shape": [
7168,
20480
],
"size": 6021120,
"type": 14
},
{
"name": "blk.59.ffn_up.weight",
"offset": 27498827776,
"shape": [
7168,
20480
],
"size": 120422400,
"type": 14
},
{
"name": "blk.59.attn_k.weight",
"offset": 27619250176,
"shape": [
7168,
1024
],
"size": 120422400,
"type": 14
},
{
"name": "blk.59.attn_output.weight",
"offset": 27625271296,
"shape": [
7168,
7168
],
"size": 6021120,
"type": 14
},
{
"name": "blk.59.attn_q.weight",
"offset": 27667419136,
"shape": [
7168,
7168
],
"size": 42147840,
"type": 14
},
{
"name": "blk.59.attn_v.weight",
"offset": 27709566976,
"shape": [
7168,
1024
],
"size": 42147840,
"type": 14
},
{
"name": "output.weight",
"offset": 27715588096,
"shape": [
7168,
64000
],
"size": 6021120,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 28091908096,
"shape": [
7168
],
"size": 376320000,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 28091936768,
"shape": [
20480,
7168
],
"size": 28672,
"type": 14
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 28212359168,
"shape": [
7168
],
"size": 120422400,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 28212387840,
"shape": [
7168
],
"size": 28672,
"type": 0
}
],
"version": 3
}