latest
80GB
The Mixtral-8x22B Large Language Model (LLM) is a pretrained generative Sparse Mixture of Experts.
8x22B
309 Pulls Updated 2 months ago
77b297b369d6 · 281GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 1,
"general.name": "models",
"llama.attention.head_count": 48,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 56,
"llama.context_length": 65536,
"llama.embedding_length": 6144,
"llama.expert_count": 8,
"llama.expert_used_count": 2,
"llama.feed_forward_length": 16384,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 1000000,
"tokenizer.ggml.add_bos_token": true,
"tokenizer.ggml.add_eos_token": false,
"tokenizer.ggml.bos_token_id": 1,
"tokenizer.ggml.eos_token_id": 2,
"tokenizer.ggml.model": "llama",
"tokenizer.ggml.scores": "... (32000 values)",
"tokenizer.ggml.token_type": "... (32000 values)",
"tokenizer.ggml.tokens": "... (32000 values)",
"tokenizer.ggml.unknown_token_id": 0
},
"num_params": 140620634112,
"tensors": [
{
"name": "blk.0.ffn_gate.0.weight",
"offset": 393216000,
"shape": [
6144,
16384
],
"size": 393216000,
"type": 1
},
{
"name": "blk.0.ffn_down.0.weight",
"offset": 594542592,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_up.0.weight",
"offset": 795869184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate.1.weight",
"offset": 997195776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_down.1.weight",
"offset": 1198522368,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_up.1.weight",
"offset": 1399848960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate.2.weight",
"offset": 1601175552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_down.2.weight",
"offset": 1802502144,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_up.2.weight",
"offset": 2003828736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate.3.weight",
"offset": 2205155328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_down.3.weight",
"offset": 2406481920,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_up.3.weight",
"offset": 2607808512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate.4.weight",
"offset": 2809135104,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_down.4.weight",
"offset": 3010461696,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_up.4.weight",
"offset": 3211788288,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate.5.weight",
"offset": 3413114880,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_down.5.weight",
"offset": 3614441472,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_up.5.weight",
"offset": 3815768064,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate.6.weight",
"offset": 4017094656,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_down.6.weight",
"offset": 4218421248,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_up.6.weight",
"offset": 4419747840,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate.7.weight",
"offset": 4621074432,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.ffn_gate_inp.weight",
"offset": 4822401024,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.attn_k.weight",
"offset": 4822499328,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.0.attn_output.weight",
"offset": 4835082240,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.0.attn_q.weight",
"offset": 4910579712,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.0.attn_v.weight",
"offset": 4986077184,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.0.ffn_down.7.weight",
"offset": 4998660096,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.0.ffn_up.7.weight",
"offset": 5199986688,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.0.attn_norm.weight",
"offset": 5401313280,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 5401337856,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.1.ffn_gate.0.weight",
"offset": 5401362432,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.1.ffn_down.0.weight",
"offset": 5602689024,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.0.weight",
"offset": 5804015616,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_gate.1.weight",
"offset": 6005342208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_down.1.weight",
"offset": 6206668800,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.1.weight",
"offset": 6407995392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_gate.2.weight",
"offset": 6609321984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_down.2.weight",
"offset": 6810648576,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.2.weight",
"offset": 7011975168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_gate.3.weight",
"offset": 7213301760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_down.3.weight",
"offset": 7414628352,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.3.weight",
"offset": 7615954944,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_gate.4.weight",
"offset": 7817281536,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_down.4.weight",
"offset": 8018608128,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.4.weight",
"offset": 8219934720,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_gate.5.weight",
"offset": 8421261312,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_down.5.weight",
"offset": 8622587904,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.5.weight",
"offset": 8823914496,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_gate.6.weight",
"offset": 9025241088,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_down.6.weight",
"offset": 9226567680,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.6.weight",
"offset": 9427894272,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_gate_inp.weight",
"offset": 9629220864,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.attn_k.weight",
"offset": 9629319168,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.1.attn_output.weight",
"offset": 9641902080,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.1.attn_q.weight",
"offset": 9717399552,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.1.attn_v.weight",
"offset": 9792897024,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.1.ffn_gate.7.weight",
"offset": 9805479936,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.1.ffn_down.7.weight",
"offset": 10006806528,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.ffn_up.7.weight",
"offset": 10208133120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.1.attn_norm.weight",
"offset": 10409459712,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 10409484288,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.2.ffn_gate.0.weight",
"offset": 10409508864,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.2.ffn_down.0.weight",
"offset": 10610835456,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_up.0.weight",
"offset": 10812162048,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_gate.1.weight",
"offset": 11013488640,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_down.1.weight",
"offset": 11214815232,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_up.1.weight",
"offset": 11416141824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_gate.2.weight",
"offset": 11617468416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_down.2.weight",
"offset": 11818795008,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_up.2.weight",
"offset": 12020121600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_gate.3.weight",
"offset": 12221448192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_down.3.weight",
"offset": 12422774784,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_up.3.weight",
"offset": 12624101376,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_gate.4.weight",
"offset": 12825427968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_down.4.weight",
"offset": 13026754560,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_up.4.weight",
"offset": 13228081152,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_gate.5.weight",
"offset": 13429407744,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_down.5.weight",
"offset": 13630734336,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_up.5.weight",
"offset": 13832060928,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_gate.6.weight",
"offset": 14033387520,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_down.6.weight",
"offset": 14234714112,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_gate_inp.weight",
"offset": 14436040704,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.attn_k.weight",
"offset": 14436139008,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.2.attn_output.weight",
"offset": 14448721920,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.2.attn_q.weight",
"offset": 14524219392,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.2.attn_v.weight",
"offset": 14599716864,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.2.ffn_up.6.weight",
"offset": 14612299776,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.2.ffn_gate.7.weight",
"offset": 14813626368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_down.7.weight",
"offset": 15014952960,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.ffn_up.7.weight",
"offset": 15216279552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.2.attn_norm.weight",
"offset": 15417606144,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 15417630720,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.3.ffn_gate.0.weight",
"offset": 15417655296,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.3.ffn_down.0.weight",
"offset": 15618981888,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_up.0.weight",
"offset": 15820308480,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate.1.weight",
"offset": 16021635072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_down.1.weight",
"offset": 16222961664,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_up.1.weight",
"offset": 16424288256,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate.2.weight",
"offset": 16625614848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_down.2.weight",
"offset": 16826941440,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_up.2.weight",
"offset": 17028268032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate.3.weight",
"offset": 17229594624,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_down.3.weight",
"offset": 17430921216,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_up.3.weight",
"offset": 17632247808,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate.4.weight",
"offset": 17833574400,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_down.4.weight",
"offset": 18034900992,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_up.4.weight",
"offset": 18236227584,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate.5.weight",
"offset": 18437554176,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_down.5.weight",
"offset": 18638880768,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_up.5.weight",
"offset": 18840207360,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate.6.weight",
"offset": 19041533952,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate_inp.weight",
"offset": 19242860544,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.attn_k.weight",
"offset": 19242958848,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.3.attn_output.weight",
"offset": 19255541760,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.3.attn_q.weight",
"offset": 19331039232,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.3.attn_v.weight",
"offset": 19406536704,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.3.ffn_down.6.weight",
"offset": 19419119616,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.3.ffn_up.6.weight",
"offset": 19620446208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_gate.7.weight",
"offset": 19821772800,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_down.7.weight",
"offset": 20023099392,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.ffn_up.7.weight",
"offset": 20224425984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.3.attn_norm.weight",
"offset": 20425752576,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 20425777152,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.4.ffn_gate.0.weight",
"offset": 20425801728,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.4.ffn_down.0.weight",
"offset": 20627128320,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.0.weight",
"offset": 20828454912,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_gate.1.weight",
"offset": 21029781504,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_down.1.weight",
"offset": 21231108096,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.1.weight",
"offset": 21432434688,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_gate.2.weight",
"offset": 21633761280,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_down.2.weight",
"offset": 21835087872,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.2.weight",
"offset": 22036414464,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_gate.3.weight",
"offset": 22237741056,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_down.3.weight",
"offset": 22439067648,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.3.weight",
"offset": 22640394240,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_gate.4.weight",
"offset": 22841720832,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_down.4.weight",
"offset": 23043047424,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.4.weight",
"offset": 23244374016,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_gate.5.weight",
"offset": 23445700608,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_down.5.weight",
"offset": 23647027200,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.5.weight",
"offset": 23848353792,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_gate_inp.weight",
"offset": 24049680384,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.attn_k.weight",
"offset": 24049778688,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.4.attn_output.weight",
"offset": 24062361600,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.4.attn_q.weight",
"offset": 24137859072,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.4.attn_v.weight",
"offset": 24213356544,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.4.ffn_gate.6.weight",
"offset": 24225939456,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.4.ffn_down.6.weight",
"offset": 24427266048,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.6.weight",
"offset": 24628592640,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_gate.7.weight",
"offset": 24829919232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_down.7.weight",
"offset": 25031245824,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.ffn_up.7.weight",
"offset": 25232572416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.4.attn_norm.weight",
"offset": 25433899008,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 25433923584,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.5.ffn_gate.0.weight",
"offset": 25433948160,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.5.ffn_down.0.weight",
"offset": 25635274752,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_up.0.weight",
"offset": 25836601344,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_gate.1.weight",
"offset": 26037927936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_down.1.weight",
"offset": 26239254528,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_up.1.weight",
"offset": 26440581120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_gate.2.weight",
"offset": 26641907712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_down.2.weight",
"offset": 26843234304,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_up.2.weight",
"offset": 27044560896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_gate.3.weight",
"offset": 27245887488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_down.3.weight",
"offset": 27447214080,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_up.3.weight",
"offset": 27648540672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_gate.4.weight",
"offset": 27849867264,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_down.4.weight",
"offset": 28051193856,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_up.4.weight",
"offset": 28252520448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_gate.5.weight",
"offset": 28453847040,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_down.5.weight",
"offset": 28655173632,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_gate_inp.weight",
"offset": 28856500224,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.attn_k.weight",
"offset": 28856598528,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.5.attn_output.weight",
"offset": 28869181440,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.5.attn_q.weight",
"offset": 28944678912,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.5.attn_v.weight",
"offset": 29020176384,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.5.ffn_up.5.weight",
"offset": 29032759296,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.5.ffn_gate.6.weight",
"offset": 29234085888,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_down.6.weight",
"offset": 29435412480,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_up.6.weight",
"offset": 29636739072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_gate.7.weight",
"offset": 29838065664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_down.7.weight",
"offset": 30039392256,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.ffn_up.7.weight",
"offset": 30240718848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.5.attn_norm.weight",
"offset": 30442045440,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 30442070016,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.6.ffn_gate.0.weight",
"offset": 30442094592,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.6.ffn_down.0.weight",
"offset": 30643421184,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_up.0.weight",
"offset": 30844747776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate.1.weight",
"offset": 31046074368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_down.1.weight",
"offset": 31247400960,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_up.1.weight",
"offset": 31448727552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate.2.weight",
"offset": 31650054144,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_down.2.weight",
"offset": 31851380736,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_up.2.weight",
"offset": 32052707328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate.3.weight",
"offset": 32254033920,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_down.3.weight",
"offset": 32455360512,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_up.3.weight",
"offset": 32656687104,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate.4.weight",
"offset": 32858013696,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_down.4.weight",
"offset": 33059340288,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_up.4.weight",
"offset": 33260666880,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate.5.weight",
"offset": 33461993472,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate_inp.weight",
"offset": 33663320064,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.attn_k.weight",
"offset": 33663418368,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.6.attn_output.weight",
"offset": 33676001280,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.6.attn_q.weight",
"offset": 33751498752,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.6.attn_v.weight",
"offset": 33826996224,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.6.ffn_down.5.weight",
"offset": 33839579136,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.6.ffn_up.5.weight",
"offset": 34040905728,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate.6.weight",
"offset": 34242232320,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_down.6.weight",
"offset": 34443558912,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_up.6.weight",
"offset": 34644885504,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_gate.7.weight",
"offset": 34846212096,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_down.7.weight",
"offset": 35047538688,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.ffn_up.7.weight",
"offset": 35248865280,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.6.attn_norm.weight",
"offset": 35450191872,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 35450216448,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.7.ffn_gate.0.weight",
"offset": 35450241024,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.7.ffn_down.0.weight",
"offset": 35651567616,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.0.weight",
"offset": 35852894208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_gate.1.weight",
"offset": 36054220800,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_down.1.weight",
"offset": 36255547392,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.1.weight",
"offset": 36456873984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_gate.2.weight",
"offset": 36658200576,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_down.2.weight",
"offset": 36859527168,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.2.weight",
"offset": 37060853760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_gate.3.weight",
"offset": 37262180352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_down.3.weight",
"offset": 37463506944,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.3.weight",
"offset": 37664833536,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_gate.4.weight",
"offset": 37866160128,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_down.4.weight",
"offset": 38067486720,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.4.weight",
"offset": 38268813312,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_gate_inp.weight",
"offset": 38470139904,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.attn_k.weight",
"offset": 38470238208,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.7.attn_output.weight",
"offset": 38482821120,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.7.attn_q.weight",
"offset": 38558318592,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.7.attn_v.weight",
"offset": 38633816064,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.7.ffn_gate.5.weight",
"offset": 38646398976,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.7.ffn_down.5.weight",
"offset": 38847725568,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.5.weight",
"offset": 39049052160,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_gate.6.weight",
"offset": 39250378752,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_down.6.weight",
"offset": 39451705344,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.6.weight",
"offset": 39653031936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_gate.7.weight",
"offset": 39854358528,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_down.7.weight",
"offset": 40055685120,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.ffn_up.7.weight",
"offset": 40257011712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.7.attn_norm.weight",
"offset": 40458338304,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 40458362880,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.8.ffn_gate.0.weight",
"offset": 40458387456,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.8.ffn_down.0.weight",
"offset": 40659714048,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_up.0.weight",
"offset": 40861040640,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_gate.1.weight",
"offset": 41062367232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_down.1.weight",
"offset": 41263693824,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_up.1.weight",
"offset": 41465020416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_gate.2.weight",
"offset": 41666347008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_down.2.weight",
"offset": 41867673600,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_up.2.weight",
"offset": 42069000192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_gate.3.weight",
"offset": 42270326784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_down.3.weight",
"offset": 42471653376,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_up.3.weight",
"offset": 42672979968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_gate.4.weight",
"offset": 42874306560,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_down.4.weight",
"offset": 43075633152,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_gate_inp.weight",
"offset": 43276959744,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.attn_k.weight",
"offset": 43277058048,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.8.attn_output.weight",
"offset": 43289640960,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.8.attn_q.weight",
"offset": 43365138432,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.8.attn_v.weight",
"offset": 43440635904,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.8.ffn_up.4.weight",
"offset": 43453218816,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.8.ffn_gate.5.weight",
"offset": 43654545408,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_down.5.weight",
"offset": 43855872000,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_up.5.weight",
"offset": 44057198592,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_gate.6.weight",
"offset": 44258525184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_down.6.weight",
"offset": 44459851776,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_up.6.weight",
"offset": 44661178368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_gate.7.weight",
"offset": 44862504960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_down.7.weight",
"offset": 45063831552,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.ffn_up.7.weight",
"offset": 45265158144,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.8.attn_norm.weight",
"offset": 45466484736,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 45466509312,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.9.ffn_gate.0.weight",
"offset": 45466533888,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.9.ffn_down.0.weight",
"offset": 45667860480,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_up.0.weight",
"offset": 45869187072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate.1.weight",
"offset": 46070513664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_down.1.weight",
"offset": 46271840256,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_up.1.weight",
"offset": 46473166848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate.2.weight",
"offset": 46674493440,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_down.2.weight",
"offset": 46875820032,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_up.2.weight",
"offset": 47077146624,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate.3.weight",
"offset": 47278473216,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_down.3.weight",
"offset": 47479799808,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_up.3.weight",
"offset": 47681126400,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate.4.weight",
"offset": 47882452992,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate_inp.weight",
"offset": 48083779584,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.attn_k.weight",
"offset": 48083877888,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.9.attn_output.weight",
"offset": 48096460800,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.9.attn_q.weight",
"offset": 48171958272,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.9.attn_v.weight",
"offset": 48247455744,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.10.ffn_gate.0.weight",
"offset": 48260038656,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.10.ffn_down.0.weight",
"offset": 48461365248,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.0.weight",
"offset": 48662691840,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_gate.1.weight",
"offset": 48864018432,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_down.1.weight",
"offset": 49065345024,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.1.weight",
"offset": 49266671616,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_gate.2.weight",
"offset": 49467998208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_down.2.weight",
"offset": 49669324800,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.2.weight",
"offset": 49870651392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_gate.3.weight",
"offset": 50071977984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_down.3.weight",
"offset": 50273304576,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.3.weight",
"offset": 50474631168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_gate_inp.weight",
"offset": 50675957760,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.attn_k.weight",
"offset": 50676056064,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.10.attn_output.weight",
"offset": 50688638976,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.10.attn_q.weight",
"offset": 50764136448,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.10.attn_v.weight",
"offset": 50839633920,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.9.ffn_down.4.weight",
"offset": 50852216832,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "weight\u0002\u0000\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0004\u0000",
"offset": 51053543424,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate.5.weight",
"offset": 51254870016,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_down.5.weight",
"offset": 51456196608,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_up.5.weight",
"offset": 51657523200,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate.6.weight",
"offset": 51858849792,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_down.6.weight",
"offset": 52060176384,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_up.6.weight",
"offset": 52261502976,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_gate.7.weight",
"offset": 52462829568,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_down.7.weight",
"offset": 52664156160,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.ffn_up.7.weight",
"offset": 52865482752,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.9.attn_norm.weight",
"offset": 53066809344,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 53066833920,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.10.ffn_gate.4.weight",
"offset": 53066858496,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.10.ffn_down.4.weight",
"offset": 53268185088,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.4.weight",
"offset": 53469511680,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_gate.5.weight",
"offset": 53670838272,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_down.5.weight",
"offset": 53872164864,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.5.weight",
"offset": 54073491456,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_gate.6.weight",
"offset": 54274818048,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_down.6.weight",
"offset": 54476144640,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.6.weight",
"offset": 54677471232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_gate.7.weight",
"offset": 54878797824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_down.7.weight",
"offset": 55080124416,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.ffn_up.7.weight",
"offset": 55281451008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.10.attn_norm.weight",
"offset": 55482777600,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 55482802176,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.11.ffn_gate.0.weight",
"offset": 55482826752,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.11.ffn_down.0.weight",
"offset": 55684153344,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_up.0.weight",
"offset": 55885479936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_gate.1.weight",
"offset": 56086806528,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_down.1.weight",
"offset": 56288133120,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_up.1.weight",
"offset": 56489459712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_gate.2.weight",
"offset": 56690786304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_down.2.weight",
"offset": 56892112896,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_up.2.weight",
"offset": 57093439488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_gate.3.weight",
"offset": 57294766080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_down.3.weight",
"offset": 57496092672,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_gate_inp.weight",
"offset": 57697419264,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.attn_k.weight",
"offset": 57697517568,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.11.attn_output.weight",
"offset": 57710100480,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.11.attn_q.weight",
"offset": 57785597952,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.11.attn_v.weight",
"offset": 57861095424,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.11.ffn_up.3.weight",
"offset": 57873678336,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.11.ffn_gate.4.weight",
"offset": 58075004928,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_down.4.weight",
"offset": 58276331520,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_up.4.weight",
"offset": 58477658112,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_gate.5.weight",
"offset": 58678984704,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_down.5.weight",
"offset": 58880311296,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_up.5.weight",
"offset": 59081637888,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_gate.6.weight",
"offset": 59282964480,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_down.6.weight",
"offset": 59484291072,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_up.6.weight",
"offset": 59685617664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_gate.7.weight",
"offset": 59886944256,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_down.7.weight",
"offset": 60088270848,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.ffn_up.7.weight",
"offset": 60289597440,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.11.attn_norm.weight",
"offset": 60490924032,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 60490948608,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.12.ffn_gate.0.weight",
"offset": 60490973184,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.12.ffn_down.0.weight",
"offset": 60692299776,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_up.0.weight",
"offset": 60893626368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate.1.weight",
"offset": 61094952960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_down.1.weight",
"offset": 61296279552,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_up.1.weight",
"offset": 61497606144,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate.2.weight",
"offset": 61698932736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_down.2.weight",
"offset": 61900259328,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_up.2.weight",
"offset": 62101585920,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate.3.weight",
"offset": 62302912512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate_inp.weight",
"offset": 62504239104,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.attn_k.weight",
"offset": 62504337408,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.12.attn_output.weight",
"offset": 62516920320,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.12.attn_q.weight",
"offset": 62592417792,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.12.attn_v.weight",
"offset": 62667915264,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.12.ffn_down.3.weight",
"offset": 62680498176,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.12.ffn_up.3.weight",
"offset": 62881824768,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate.4.weight",
"offset": 63083151360,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_down.4.weight",
"offset": 63284477952,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_up.4.weight",
"offset": 63485804544,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate.5.weight",
"offset": 63687131136,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_down.5.weight",
"offset": 63888457728,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_up.5.weight",
"offset": 64089784320,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate.6.weight",
"offset": 64291110912,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_down.6.weight",
"offset": 64492437504,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_up.6.weight",
"offset": 64693764096,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_gate.7.weight",
"offset": 64895090688,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_down.7.weight",
"offset": 65096417280,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.ffn_up.7.weight",
"offset": 65297743872,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.12.attn_norm.weight",
"offset": 65499070464,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 65499095040,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.13.ffn_gate.0.weight",
"offset": 65499119616,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.13.ffn_down.0.weight",
"offset": 65700446208,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.0.weight",
"offset": 65901772800,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_gate.1.weight",
"offset": 66103099392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_down.1.weight",
"offset": 66304425984,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.1.weight",
"offset": 66505752576,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_gate.2.weight",
"offset": 66707079168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_down.2.weight",
"offset": 66908405760,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.2.weight",
"offset": 67109732352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_gate_inp.weight",
"offset": 67311058944,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.attn_k.weight",
"offset": 67311157248,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.13.attn_output.weight",
"offset": 67323740160,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.13.attn_q.weight",
"offset": 67399237632,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.13.attn_v.weight",
"offset": 67474735104,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.13.ffn_gate.3.weight",
"offset": 67487318016,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.13.ffn_down.3.weight",
"offset": 67688644608,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.3.weight",
"offset": 67889971200,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_gate.4.weight",
"offset": 68091297792,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_down.4.weight",
"offset": 68292624384,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.4.weight",
"offset": 68493950976,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_gate.5.weight",
"offset": 68695277568,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_down.5.weight",
"offset": 68896604160,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.5.weight",
"offset": 69097930752,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_gate.6.weight",
"offset": 69299257344,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_down.6.weight",
"offset": 69500583936,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.6.weight",
"offset": 69701910528,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_gate.7.weight",
"offset": 69903237120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_down.7.weight",
"offset": 70104563712,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.ffn_up.7.weight",
"offset": 70305890304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.13.attn_norm.weight",
"offset": 70507216896,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 70507241472,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.14.ffn_gate.0.weight",
"offset": 70507266048,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.14.ffn_down.0.weight",
"offset": 70708592640,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_up.0.weight",
"offset": 70909919232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_gate.1.weight",
"offset": 71111245824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_down.1.weight",
"offset": 71312572416,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_up.1.weight",
"offset": 71513899008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_gate.2.weight",
"offset": 71715225600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_down.2.weight",
"offset": 71916552192,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_gate_inp.weight",
"offset": 72117878784,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.attn_k.weight",
"offset": 72117977088,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.14.attn_output.weight",
"offset": 72130560000,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.14.attn_q.weight",
"offset": 72206057472,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.14.attn_v.weight",
"offset": 72281554944,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.14.ffn_up.2.weight",
"offset": 72294137856,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.14.ffn_gate.3.weight",
"offset": 72495464448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_down.3.weight",
"offset": 72696791040,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_up.3.weight",
"offset": 72898117632,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_gate.4.weight",
"offset": 73099444224,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_down.4.weight",
"offset": 73300770816,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_up.4.weight",
"offset": 73502097408,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_gate.5.weight",
"offset": 73703424000,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_down.5.weight",
"offset": 73904750592,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_up.5.weight",
"offset": 74106077184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_gate.6.weight",
"offset": 74307403776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_down.6.weight",
"offset": 74508730368,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_up.6.weight",
"offset": 74710056960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_gate.7.weight",
"offset": 74911383552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_down.7.weight",
"offset": 75112710144,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.ffn_up.7.weight",
"offset": 75314036736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.14.attn_norm.weight",
"offset": 75515363328,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 75515387904,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.15.ffn_gate.0.weight",
"offset": 75515412480,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.15.ffn_down.0.weight",
"offset": 75716739072,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_up.0.weight",
"offset": 75918065664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate.1.weight",
"offset": 76119392256,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_down.1.weight",
"offset": 76320718848,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_up.1.weight",
"offset": 76522045440,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate.2.weight",
"offset": 76723372032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate_inp.weight",
"offset": 76924698624,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.attn_k.weight",
"offset": 76924796928,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.15.attn_output.weight",
"offset": 76937379840,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.15.attn_q.weight",
"offset": 77012877312,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.15.attn_v.weight",
"offset": 77088374784,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.15.ffn_down.2.weight",
"offset": 77100957696,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.15.ffn_up.2.weight",
"offset": 77302284288,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate.3.weight",
"offset": 77503610880,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_down.3.weight",
"offset": 77704937472,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_up.3.weight",
"offset": 77906264064,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate.4.weight",
"offset": 78107590656,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_down.4.weight",
"offset": 78308917248,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_up.4.weight",
"offset": 78510243840,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate.5.weight",
"offset": 78711570432,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_down.5.weight",
"offset": 78912897024,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_up.5.weight",
"offset": 79114223616,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate.6.weight",
"offset": 79315550208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_down.6.weight",
"offset": 79516876800,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_up.6.weight",
"offset": 79718203392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_gate.7.weight",
"offset": 79919529984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_down.7.weight",
"offset": 80120856576,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.ffn_up.7.weight",
"offset": 80322183168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.15.attn_norm.weight",
"offset": 80523509760,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 80523534336,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.16.ffn_gate.0.weight",
"offset": 80523558912,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.16.ffn_down.0.weight",
"offset": 80724885504,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.0.weight",
"offset": 80926212096,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_gate.1.weight",
"offset": 81127538688,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_down.1.weight",
"offset": 81328865280,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.1.weight",
"offset": 81530191872,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_gate_inp.weight",
"offset": 81731518464,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.attn_k.weight",
"offset": 81731616768,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.16.attn_output.weight",
"offset": 81744199680,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.16.attn_q.weight",
"offset": 81819697152,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.16.attn_v.weight",
"offset": 81895194624,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.16.ffn_gate.2.weight",
"offset": 81907777536,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.16.ffn_down.2.weight",
"offset": 82109104128,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.2.weight",
"offset": 82310430720,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_gate.3.weight",
"offset": 82511757312,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_down.3.weight",
"offset": 82713083904,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.3.weight",
"offset": 82914410496,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_gate.4.weight",
"offset": 83115737088,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_down.4.weight",
"offset": 83317063680,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.4.weight",
"offset": 83518390272,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_gate.5.weight",
"offset": 83719716864,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_down.5.weight",
"offset": 83921043456,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.5.weight",
"offset": 84122370048,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_gate.6.weight",
"offset": 84323696640,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_down.6.weight",
"offset": 84525023232,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.6.weight",
"offset": 84726349824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_gate.7.weight",
"offset": 84927676416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_down.7.weight",
"offset": 85129003008,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.ffn_up.7.weight",
"offset": 85330329600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.16.attn_norm.weight",
"offset": 85531656192,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 85531680768,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.17.ffn_gate.0.weight",
"offset": 85531705344,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.17.ffn_down.0.weight",
"offset": 85733031936,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_up.0.weight",
"offset": 85934358528,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_gate.1.weight",
"offset": 86135685120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_down.1.weight",
"offset": 86337011712,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_gate_inp.weight",
"offset": 86538338304,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.attn_k.weight",
"offset": 86538436608,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.17.attn_output.weight",
"offset": 86551019520,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.17.attn_q.weight",
"offset": 86626516992,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.17.attn_v.weight",
"offset": 86702014464,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.17.ffn_up.1.weight",
"offset": 86714597376,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.17.ffn_gate.2.weight",
"offset": 86915923968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_down.2.weight",
"offset": 87117250560,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_up.2.weight",
"offset": 87318577152,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_gate.3.weight",
"offset": 87519903744,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_down.3.weight",
"offset": 87721230336,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_up.3.weight",
"offset": 87922556928,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_gate.4.weight",
"offset": 88123883520,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_down.4.weight",
"offset": 88325210112,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_up.4.weight",
"offset": 88526536704,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_gate.5.weight",
"offset": 88727863296,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_down.5.weight",
"offset": 88929189888,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_up.5.weight",
"offset": 89130516480,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_gate.6.weight",
"offset": 89331843072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_down.6.weight",
"offset": 89533169664,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_up.6.weight",
"offset": 89734496256,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_gate.7.weight",
"offset": 89935822848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_down.7.weight",
"offset": 90137149440,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.ffn_up.7.weight",
"offset": 90338476032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.17.attn_norm.weight",
"offset": 90539802624,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 90539827200,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.18.ffn_gate.0.weight",
"offset": 90539851776,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.18.ffn_down.0.weight",
"offset": 90741178368,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_up.0.weight",
"offset": 90942504960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate.1.weight",
"offset": 91143831552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate_inp.weight",
"offset": 91345158144,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.attn_k.weight",
"offset": 91345256448,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.18.attn_output.weight",
"offset": 91357839360,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.18.attn_q.weight",
"offset": 91433336832,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.18.attn_v.weight",
"offset": 91508834304,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.18.ffn_down.1.weight",
"offset": 91521417216,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.18.ffn_up.1.weight",
"offset": 91722743808,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate.2.weight",
"offset": 91924070400,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_down.2.weight",
"offset": 92125396992,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_up.2.weight",
"offset": 92326723584,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate.3.weight",
"offset": 92528050176,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_down.3.weight",
"offset": 92729376768,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_up.3.weight",
"offset": 92930703360,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate.4.weight",
"offset": 93132029952,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_down.4.weight",
"offset": 93333356544,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "\u0000\u0000#\ufffd\u001a\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.22",
"offset": 93534683136,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate.5.weight",
"offset": 93736009728,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_down.5.weight",
"offset": 93937336320,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_up.5.weight",
"offset": 94138662912,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate.6.weight",
"offset": 94339989504,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_down.6.weight",
"offset": 94541316096,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_up.6.weight",
"offset": 94742642688,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_gate.7.weight",
"offset": 94943969280,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_down.7.weight",
"offset": 95145295872,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.ffn_up.7.weight",
"offset": 95346622464,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.18.attn_norm.weight",
"offset": 95547949056,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 95547973632,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.19.ffn_gate.0.weight",
"offset": 95547998208,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.19.ffn_down.0.weight",
"offset": 95749324800,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.0.weight",
"offset": 95950651392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_gate_inp.weight",
"offset": 96151977984,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.attn_k.weight",
"offset": 96152076288,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.19.attn_output.weight",
"offset": 96164659200,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.19.attn_q.weight",
"offset": 96240156672,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.19.attn_v.weight",
"offset": 96315654144,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.19.ffn_gate.1.weight",
"offset": 96328237056,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.19.ffn_down.1.weight",
"offset": 96529563648,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.1.weight",
"offset": 96730890240,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_gate.2.weight",
"offset": 96932216832,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_down.2.weight",
"offset": 97133543424,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.2.weight",
"offset": 97334870016,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_gate.3.weight",
"offset": 97536196608,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_down.3.weight",
"offset": 97737523200,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.3.weight",
"offset": 97938849792,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_gate.4.weight",
"offset": 98140176384,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_down.4.weight",
"offset": 98341502976,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.4.weight",
"offset": 98542829568,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_gate.5.weight",
"offset": 98744156160,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_down.5.weight",
"offset": 98945482752,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.5.weight",
"offset": 99146809344,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_gate.6.weight",
"offset": 99348135936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_down.6.weight",
"offset": 99549462528,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.6.weight",
"offset": 99750789120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_gate.7.weight",
"offset": 99952115712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_down.7.weight",
"offset": 100153442304,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.ffn_up.7.weight",
"offset": 100354768896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.19.attn_norm.weight",
"offset": 100556095488,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 100556120064,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.20.ffn_gate.0.weight",
"offset": 100556144640,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.20.ffn_down.0.weight",
"offset": 100757471232,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_gate_inp.weight",
"offset": 100958797824,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.attn_k.weight",
"offset": 100958896128,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.20.attn_output.weight",
"offset": 100971479040,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.20.attn_q.weight",
"offset": 101046976512,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.20.attn_v.weight",
"offset": 101122473984,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.20.ffn_up.0.weight",
"offset": 101135056896,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.20.ffn_gate.1.weight",
"offset": 101336383488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_down.1.weight",
"offset": 101537710080,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_up.1.weight",
"offset": 101739036672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_gate.2.weight",
"offset": 101940363264,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_down.2.weight",
"offset": 102141689856,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_up.2.weight",
"offset": 102343016448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_gate.3.weight",
"offset": 102544343040,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_down.3.weight",
"offset": 102745669632,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_up.3.weight",
"offset": 102946996224,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_gate.4.weight",
"offset": 103148322816,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_down.4.weight",
"offset": 103349649408,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_up.4.weight",
"offset": 103550976000,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_gate.5.weight",
"offset": 103752302592,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_down.5.weight",
"offset": 103953629184,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_up.5.weight",
"offset": 104154955776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_gate.6.weight",
"offset": 104356282368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_down.6.weight",
"offset": 104557608960,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_up.6.weight",
"offset": 104758935552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_gate.7.weight",
"offset": 104960262144,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_down.7.weight",
"offset": 105161588736,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.ffn_up.7.weight",
"offset": 105362915328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.20.attn_norm.weight",
"offset": 105564241920,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 105564266496,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.21.ffn_gate.0.weight",
"offset": 105564291072,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.21.ffn_gate_inp.weight",
"offset": 105765617664,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.attn_k.weight",
"offset": 105765715968,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.21.attn_output.weight",
"offset": 105778298880,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.21.attn_q.weight",
"offset": 105853796352,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.21.attn_v.weight",
"offset": 105929293824,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.21.ffn_down.0.weight",
"offset": 105941876736,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.21.ffn_up.0.weight",
"offset": 106143203328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_gate.1.weight",
"offset": 106344529920,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_down.1.weight",
"offset": 106545856512,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_up.1.weight",
"offset": 106747183104,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_gate.2.weight",
"offset": 106948509696,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_down.2.weight",
"offset": 107149836288,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_up.2.weight",
"offset": 107351162880,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_gate.3.weight",
"offset": 107552489472,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_down.3.weight",
"offset": 107753816064,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_up.3.weight",
"offset": 107955142656,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_gate.4.weight",
"offset": 108156469248,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_down.4.weight",
"offset": 108357795840,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_up.4.weight",
"offset": 108559122432,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_gate.5.weight",
"offset": 108760449024,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_down.5.weight",
"offset": 108961775616,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_up.5.weight",
"offset": 109163102208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_gate.6.weight",
"offset": 109364428800,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_down.6.weight",
"offset": 109565755392,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_up.6.weight",
"offset": 109767081984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_gate.7.weight",
"offset": 109968408576,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_down.7.weight",
"offset": 110169735168,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.ffn_up.7.weight",
"offset": 110371061760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.21.attn_norm.weight",
"offset": 110572388352,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 110572412928,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.22.ffn_gate_inp.weight",
"offset": 110572437504,
"shape": [
6144,
8
],
"size": 24576,
"type": 1
},
{
"name": "blk.22.attn_k.weight",
"offset": 110572535808,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.22.attn_output.weight",
"offset": 110585118720,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.22.attn_q.weight",
"offset": 110660616192,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.22.attn_v.weight",
"offset": 110736113664,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.22.ffn_gate.0.weight",
"offset": 110748696576,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.22.ffn_down.0.weight",
"offset": 110950023168,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.0.weight",
"offset": 111151349760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_gate.1.weight",
"offset": 111352676352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_down.1.weight",
"offset": 111554002944,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.1.weight",
"offset": 111755329536,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_gate.2.weight",
"offset": 111956656128,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_down.2.weight",
"offset": 112157982720,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.2.weight",
"offset": 112359309312,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_gate.3.weight",
"offset": 112560635904,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_down.3.weight",
"offset": 112761962496,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.3.weight",
"offset": 112963289088,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_gate.4.weight",
"offset": 113164615680,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_down.4.weight",
"offset": 113365942272,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.4.weight",
"offset": 113567268864,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_gate.5.weight",
"offset": 113768595456,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_down.5.weight",
"offset": 113969922048,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.5.weight",
"offset": 114171248640,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_gate.6.weight",
"offset": 114372575232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_down.6.weight",
"offset": 114573901824,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.6.weight",
"offset": 114775228416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_gate.7.weight",
"offset": 114976555008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_down.7.weight",
"offset": 115177881600,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.ffn_up.7.weight",
"offset": 115379208192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.22.attn_norm.weight",
"offset": 115580534784,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 115580559360,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 115580583936,
"shape": [
6144,
1024
],
"size": 24576,
"type": 1
},
{
"name": "blk.23.attn_q.weight",
"offset": 115593166848,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.23.attn_v.weight",
"offset": 115668664320,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.23.ffn_gate.0.weight",
"offset": 115681247232,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.23.ffn_down.0.weight",
"offset": 115882573824,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.0.weight",
"offset": 116083900416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate.1.weight",
"offset": 116285227008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_down.1.weight",
"offset": 116486553600,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.1.weight",
"offset": 116687880192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate.2.weight",
"offset": 116889206784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_down.2.weight",
"offset": 117090533376,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.2.weight",
"offset": 117291859968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate.3.weight",
"offset": 117493186560,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_down.3.weight",
"offset": 117694513152,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.3.weight",
"offset": 117895839744,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate.4.weight",
"offset": 118097166336,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_down.4.weight",
"offset": 118298492928,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.4.weight",
"offset": 118499819520,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate.5.weight",
"offset": 118701146112,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_down.5.weight",
"offset": 118902472704,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.5.weight",
"offset": 119103799296,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate.6.weight",
"offset": 119305125888,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_down.6.weight",
"offset": 119506452480,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.6.weight",
"offset": 119707779072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate.7.weight",
"offset": 119909105664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_down.7.weight",
"offset": 120110432256,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_up.7.weight",
"offset": 120311758848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.ffn_gate_inp.weight",
"offset": 120513085440,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.23.attn_norm.weight",
"offset": 120513183744,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 120513208320,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.23.attn_output.weight",
"offset": 120513232896,
"shape": [
6144,
6144
],
"size": 24576,
"type": 1
},
{
"name": "blk.24.attn_k.weight",
"offset": 120588730368,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.24.attn_q.weight",
"offset": 120601313280,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.24.ffn_gate.0.weight",
"offset": 120676810752,
"shape": [
6144,
16384
],
"size": 75497472,
"type": 1
},
{
"name": "blk.24.ffn_down.0.weight",
"offset": 120878137344,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.0.weight",
"offset": 121079463936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate.1.weight",
"offset": 121280790528,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_down.1.weight",
"offset": 121482117120,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.1.weight",
"offset": 121683443712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate.2.weight",
"offset": 121884770304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_down.2.weight",
"offset": 122086096896,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.2.weight",
"offset": 122287423488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate.3.weight",
"offset": 122488750080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_down.3.weight",
"offset": 122690076672,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.3.weight",
"offset": 122891403264,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate.4.weight",
"offset": 123092729856,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_down.4.weight",
"offset": 123294056448,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.4.weight",
"offset": 123495383040,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate.5.weight",
"offset": 123696709632,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_down.5.weight",
"offset": 123898036224,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.5.weight",
"offset": 124099362816,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate.6.weight",
"offset": 124300689408,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_down.6.weight",
"offset": 124502016000,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.6.weight",
"offset": 124703342592,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate.7.weight",
"offset": 124904669184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_down.7.weight",
"offset": 125105995776,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_up.7.weight",
"offset": 125307322368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.ffn_gate_inp.weight",
"offset": 125508648960,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.24.attn_norm.weight",
"offset": 125508747264,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 125508771840,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.24.attn_output.weight",
"offset": 125508796416,
"shape": [
6144,
6144
],
"size": 24576,
"type": 1
},
{
"name": "blk.24.attn_v.weight",
"offset": 125584293888,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.25.attn_q.weight",
"offset": 125596876800,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.25.ffn_gate.0.weight",
"offset": 125672374272,
"shape": [
6144,
16384
],
"size": 75497472,
"type": 1
},
{
"name": "blk.25.ffn_down.0.weight",
"offset": 125873700864,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.0.weight",
"offset": 126075027456,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate.1.weight",
"offset": 126276354048,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_down.1.weight",
"offset": 126477680640,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.1.weight",
"offset": 126679007232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate.2.weight",
"offset": 126880333824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_down.2.weight",
"offset": 127081660416,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.2.weight",
"offset": 127282987008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate.3.weight",
"offset": 127484313600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_down.3.weight",
"offset": 127685640192,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.3.weight",
"offset": 127886966784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate.4.weight",
"offset": 128088293376,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_down.4.weight",
"offset": 128289619968,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.4.weight",
"offset": 128490946560,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate.5.weight",
"offset": 128692273152,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_down.5.weight",
"offset": 128893599744,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.5.weight",
"offset": 129094926336,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate.6.weight",
"offset": 129296252928,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_down.6.weight",
"offset": 129497579520,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.6.weight",
"offset": 129698906112,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate.7.weight",
"offset": 129900232704,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_down.7.weight",
"offset": 130101559296,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_up.7.weight",
"offset": 130302885888,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.ffn_gate_inp.weight",
"offset": 130504212480,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.25.attn_norm.weight",
"offset": 130504310784,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 130504335360,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 130504359936,
"shape": [
6144,
1024
],
"size": 24576,
"type": 1
},
{
"name": "blk.25.attn_output.weight",
"offset": 130516942848,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.25.attn_v.weight",
"offset": 130592440320,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.26.ffn_gate.0.weight",
"offset": 130605023232,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.26.ffn_down.0.weight",
"offset": 130806349824,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_up.0.weight",
"offset": 131007676416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate.1.weight",
"offset": 131209003008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_down.1.weight",
"offset": 131410329600,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_up.1.weight",
"offset": 131611656192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate.2.weight",
"offset": 131812982784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_down.2.weight",
"offset": 132014309376,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_up.2.weight",
"offset": 132215635968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate.3.weight",
"offset": 132416962560,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_down.3.weight",
"offset": 132618289152,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_up.3.weight",
"offset": 132819615744,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate.4.weight",
"offset": 133020942336,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_down.4.weight",
"offset": 133222268928,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_up.4.weight",
"offset": 133423595520,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate.5.weight",
"offset": 133624922112,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_down.5.weight",
"offset": 133826248704,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_up.5.weight",
"offset": 134027575296,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate.6.weight",
"offset": 134228901888,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_down.6.weight",
"offset": 134430228480,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_up.6.weight",
"offset": 134631555072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate.7.weight",
"offset": 134832881664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_down.7.weight",
"offset": 135034208256,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.ffn_gate_inp.weight",
"offset": 135235534848,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.26.attn_k.weight",
"offset": 135235633152,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.26.attn_output.weight",
"offset": 135248216064,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.26.attn_q.weight",
"offset": 135323713536,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.26.attn_v.weight",
"offset": 135399211008,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.26.ffn_up.7.weight",
"offset": 135411793920,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.26.attn_norm.weight",
"offset": 135613120512,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 135613145088,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.27.ffn_gate.0.weight",
"offset": 135613169664,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.27.ffn_down.0.weight",
"offset": 135814496256,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "\u0000\u0000\ufffd5\ufffd$\u0000\u0000\u0000\u0018\u0000\u0000\u0000\u0000\u0000\u0000\u0000blk.3",
"offset": 136015822848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate.1.weight",
"offset": 136217149440,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_down.1.weight",
"offset": 136418476032,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_up.1.weight",
"offset": 136619802624,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate.2.weight",
"offset": 136821129216,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_down.2.weight",
"offset": 137022455808,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_up.2.weight",
"offset": 137223782400,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate.3.weight",
"offset": 137425108992,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_down.3.weight",
"offset": 137626435584,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_up.3.weight",
"offset": 137827762176,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate.4.weight",
"offset": 138029088768,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_down.4.weight",
"offset": 138230415360,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_up.4.weight",
"offset": 138431741952,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate.5.weight",
"offset": 138633068544,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_down.5.weight",
"offset": 138834395136,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_up.5.weight",
"offset": 139035721728,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate.6.weight",
"offset": 139237048320,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_down.6.weight",
"offset": 139438374912,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_up.6.weight",
"offset": 139639701504,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate.7.weight",
"offset": 139841028096,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.ffn_gate_inp.weight",
"offset": 140042354688,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.attn_k.weight",
"offset": 140042452992,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.27.attn_output.weight",
"offset": 140055035904,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.27.attn_q.weight",
"offset": 140130533376,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.27.attn_v.weight",
"offset": 140206030848,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.27.ffn_down.7.weight",
"offset": 140218613760,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.27.ffn_up.7.weight",
"offset": 140419940352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.27.attn_norm.weight",
"offset": 140621266944,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 140621291520,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.28.ffn_gate.0.weight",
"offset": 140621316096,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.28.ffn_down.0.weight",
"offset": 140822642688,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.0.weight",
"offset": 141023969280,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_gate.1.weight",
"offset": 141225295872,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_down.1.weight",
"offset": 141426622464,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.1.weight",
"offset": 141627949056,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_gate.2.weight",
"offset": 141829275648,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_down.2.weight",
"offset": 142030602240,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.2.weight",
"offset": 142231928832,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_gate.3.weight",
"offset": 142433255424,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_down.3.weight",
"offset": 142634582016,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.3.weight",
"offset": 142835908608,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_gate.4.weight",
"offset": 143037235200,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_down.4.weight",
"offset": 143238561792,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.4.weight",
"offset": 143439888384,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_gate.5.weight",
"offset": 143641214976,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_down.5.weight",
"offset": 143842541568,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.5.weight",
"offset": 144043868160,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_gate.6.weight",
"offset": 144245194752,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_down.6.weight",
"offset": 144446521344,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.6.weight",
"offset": 144647847936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_gate_inp.weight",
"offset": 144849174528,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.attn_k.weight",
"offset": 144849272832,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.28.attn_output.weight",
"offset": 144861855744,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.28.attn_q.weight",
"offset": 144937353216,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.28.attn_v.weight",
"offset": 145012850688,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.28.ffn_gate.7.weight",
"offset": 145025433600,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.28.ffn_down.7.weight",
"offset": 145226760192,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.ffn_up.7.weight",
"offset": 145428086784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.28.attn_norm.weight",
"offset": 145629413376,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 145629437952,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.29.ffn_gate.0.weight",
"offset": 145629462528,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.29.ffn_down.0.weight",
"offset": 145830789120,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_up.0.weight",
"offset": 146032115712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_gate.1.weight",
"offset": 146233442304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_down.1.weight",
"offset": 146434768896,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_up.1.weight",
"offset": 146636095488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_gate.2.weight",
"offset": 146837422080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_down.2.weight",
"offset": 147038748672,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_up.2.weight",
"offset": 147240075264,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_gate.3.weight",
"offset": 147441401856,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_down.3.weight",
"offset": 147642728448,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_up.3.weight",
"offset": 147844055040,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_gate.4.weight",
"offset": 148045381632,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_down.4.weight",
"offset": 148246708224,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_up.4.weight",
"offset": 148448034816,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_gate.5.weight",
"offset": 148649361408,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_down.5.weight",
"offset": 148850688000,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_up.5.weight",
"offset": 149052014592,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_gate.6.weight",
"offset": 149253341184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_down.6.weight",
"offset": 149454667776,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_gate_inp.weight",
"offset": 149655994368,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.attn_k.weight",
"offset": 149656092672,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.29.attn_output.weight",
"offset": 149668675584,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.29.attn_q.weight",
"offset": 149744173056,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.29.attn_v.weight",
"offset": 149819670528,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.29.ffn_up.6.weight",
"offset": 149832253440,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.29.ffn_gate.7.weight",
"offset": 150033580032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_down.7.weight",
"offset": 150234906624,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.ffn_up.7.weight",
"offset": 150436233216,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.29.attn_norm.weight",
"offset": 150637559808,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 150637584384,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.30.ffn_gate.0.weight",
"offset": 150637608960,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.30.ffn_down.0.weight",
"offset": 150838935552,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_up.0.weight",
"offset": 151040262144,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate.1.weight",
"offset": 151241588736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_down.1.weight",
"offset": 151442915328,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_up.1.weight",
"offset": 151644241920,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate.2.weight",
"offset": 151845568512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_down.2.weight",
"offset": 152046895104,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_up.2.weight",
"offset": 152248221696,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate.3.weight",
"offset": 152449548288,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_down.3.weight",
"offset": 152650874880,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_up.3.weight",
"offset": 152852201472,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate.4.weight",
"offset": 153053528064,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_down.4.weight",
"offset": 153254854656,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_up.4.weight",
"offset": 153456181248,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate.5.weight",
"offset": 153657507840,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_down.5.weight",
"offset": 153858834432,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_up.5.weight",
"offset": 154060161024,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate.6.weight",
"offset": 154261487616,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate_inp.weight",
"offset": 154462814208,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.attn_k.weight",
"offset": 154462912512,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.30.attn_output.weight",
"offset": 154475495424,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.30.attn_q.weight",
"offset": 154550992896,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.30.attn_v.weight",
"offset": 154626490368,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.30.ffn_down.6.weight",
"offset": 154639073280,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.30.ffn_up.6.weight",
"offset": 154840399872,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_gate.7.weight",
"offset": 155041726464,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_down.7.weight",
"offset": 155243053056,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.ffn_up.7.weight",
"offset": 155444379648,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.30.attn_norm.weight",
"offset": 155645706240,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 155645730816,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.31.ffn_gate.0.weight",
"offset": 155645755392,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.31.ffn_down.0.weight",
"offset": 155847081984,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.0.weight",
"offset": 156048408576,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_gate.1.weight",
"offset": 156249735168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_down.1.weight",
"offset": 156451061760,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.1.weight",
"offset": 156652388352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_gate.2.weight",
"offset": 156853714944,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_down.2.weight",
"offset": 157055041536,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.2.weight",
"offset": 157256368128,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_gate.3.weight",
"offset": 157457694720,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_down.3.weight",
"offset": 157659021312,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.3.weight",
"offset": 157860347904,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_gate.4.weight",
"offset": 158061674496,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_down.4.weight",
"offset": 158263001088,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.4.weight",
"offset": 158464327680,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_gate.5.weight",
"offset": 158665654272,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_down.5.weight",
"offset": 158866980864,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.5.weight",
"offset": 159068307456,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_gate_inp.weight",
"offset": 159269634048,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.attn_k.weight",
"offset": 159269732352,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.31.attn_output.weight",
"offset": 159282315264,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.31.attn_q.weight",
"offset": 159357812736,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.31.attn_v.weight",
"offset": 159433310208,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.31.ffn_gate.6.weight",
"offset": 159445893120,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.31.ffn_down.6.weight",
"offset": 159647219712,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.6.weight",
"offset": 159848546304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_gate.7.weight",
"offset": 160049872896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_down.7.weight",
"offset": 160251199488,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.ffn_up.7.weight",
"offset": 160452526080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.31.attn_norm.weight",
"offset": 160653852672,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 160653877248,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.32.ffn_gate.0.weight",
"offset": 160653901824,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.32.ffn_down.0.weight",
"offset": 160855228416,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_up.0.weight",
"offset": 161056555008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_gate.1.weight",
"offset": 161257881600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_down.1.weight",
"offset": 161459208192,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_up.1.weight",
"offset": 161660534784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_gate.2.weight",
"offset": 161861861376,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_down.2.weight",
"offset": 162063187968,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_up.2.weight",
"offset": 162264514560,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_gate.3.weight",
"offset": 162465841152,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_down.3.weight",
"offset": 162667167744,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_up.3.weight",
"offset": 162868494336,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_gate.4.weight",
"offset": 163069820928,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_down.4.weight",
"offset": 163271147520,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_up.4.weight",
"offset": 163472474112,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_gate.5.weight",
"offset": 163673800704,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_down.5.weight",
"offset": 163875127296,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_gate_inp.weight",
"offset": 164076453888,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.attn_k.weight",
"offset": 164076552192,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.32.attn_output.weight",
"offset": 164089135104,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.32.attn_q.weight",
"offset": 164164632576,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.32.attn_v.weight",
"offset": 164240130048,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.32.ffn_up.5.weight",
"offset": 164252712960,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.32.ffn_gate.6.weight",
"offset": 164454039552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_down.6.weight",
"offset": 164655366144,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_up.6.weight",
"offset": 164856692736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_gate.7.weight",
"offset": 165058019328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_down.7.weight",
"offset": 165259345920,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.ffn_up.7.weight",
"offset": 165460672512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.32.attn_norm.weight",
"offset": 165661999104,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 165662023680,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.33.ffn_gate.0.weight",
"offset": 165662048256,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.33.ffn_down.0.weight",
"offset": 165863374848,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_up.0.weight",
"offset": 166064701440,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate.1.weight",
"offset": 166266028032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_down.1.weight",
"offset": 166467354624,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_up.1.weight",
"offset": 166668681216,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate.2.weight",
"offset": 166870007808,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_down.2.weight",
"offset": 167071334400,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_up.2.weight",
"offset": 167272660992,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate.3.weight",
"offset": 167473987584,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_down.3.weight",
"offset": 167675314176,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_up.3.weight",
"offset": 167876640768,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate.4.weight",
"offset": 168077967360,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_down.4.weight",
"offset": 168279293952,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_up.4.weight",
"offset": 168480620544,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate.5.weight",
"offset": 168681947136,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate_inp.weight",
"offset": 168883273728,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.attn_k.weight",
"offset": 168883372032,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.33.attn_output.weight",
"offset": 168895954944,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.33.attn_q.weight",
"offset": 168971452416,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.33.attn_v.weight",
"offset": 169046949888,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.33.ffn_down.5.weight",
"offset": 169059532800,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.33.ffn_up.5.weight",
"offset": 169260859392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate.6.weight",
"offset": 169462185984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_down.6.weight",
"offset": 169663512576,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_up.6.weight",
"offset": 169864839168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_gate.7.weight",
"offset": 170066165760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_down.7.weight",
"offset": 170267492352,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.ffn_up.7.weight",
"offset": 170468818944,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.33.attn_norm.weight",
"offset": 170670145536,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 170670170112,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.34.ffn_gate.0.weight",
"offset": 170670194688,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.34.ffn_down.0.weight",
"offset": 170871521280,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.0.weight",
"offset": 171072847872,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_gate.1.weight",
"offset": 171274174464,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_down.1.weight",
"offset": 171475501056,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.1.weight",
"offset": 171676827648,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_gate.2.weight",
"offset": 171878154240,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_down.2.weight",
"offset": 172079480832,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.2.weight",
"offset": 172280807424,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_gate.3.weight",
"offset": 172482134016,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_down.3.weight",
"offset": 172683460608,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.3.weight",
"offset": 172884787200,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_gate.4.weight",
"offset": 173086113792,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_down.4.weight",
"offset": 173287440384,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.4.weight",
"offset": 173488766976,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_gate_inp.weight",
"offset": 173690093568,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.attn_k.weight",
"offset": 173690191872,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.34.attn_output.weight",
"offset": 173702774784,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.34.attn_q.weight",
"offset": 173778272256,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.34.attn_v.weight",
"offset": 173853769728,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.34.ffn_gate.5.weight",
"offset": 173866352640,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.34.ffn_down.5.weight",
"offset": 174067679232,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.5.weight",
"offset": 174269005824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_gate.6.weight",
"offset": 174470332416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_down.6.weight",
"offset": 174671659008,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.6.weight",
"offset": 174872985600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_gate.7.weight",
"offset": 175074312192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_down.7.weight",
"offset": 175275638784,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.ffn_up.7.weight",
"offset": 175476965376,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.34.attn_norm.weight",
"offset": 175678291968,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 175678316544,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.35.ffn_gate.0.weight",
"offset": 175678341120,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.35.ffn_down.0.weight",
"offset": 175879667712,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_up.0.weight",
"offset": 176080994304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_gate.1.weight",
"offset": 176282320896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_down.1.weight",
"offset": 176483647488,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_up.1.weight",
"offset": 176684974080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_gate.2.weight",
"offset": 176886300672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_down.2.weight",
"offset": 177087627264,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_up.2.weight",
"offset": 177288953856,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_gate.3.weight",
"offset": 177490280448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_down.3.weight",
"offset": 177691607040,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_up.3.weight",
"offset": 177892933632,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_gate.4.weight",
"offset": 178094260224,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_down.4.weight",
"offset": 178295586816,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_gate_inp.weight",
"offset": 178496913408,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.attn_k.weight",
"offset": 178497011712,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.35.attn_output.weight",
"offset": 178509594624,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.35.attn_q.weight",
"offset": 178585092096,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.35.attn_v.weight",
"offset": 178660589568,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.35.ffn_up.4.weight",
"offset": 178673172480,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.35.ffn_gate.5.weight",
"offset": 178874499072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_down.5.weight",
"offset": 179075825664,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_up.5.weight",
"offset": 179277152256,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_gate.6.weight",
"offset": 179478478848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_down.6.weight",
"offset": 179679805440,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_up.6.weight",
"offset": 179881132032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_gate.7.weight",
"offset": 180082458624,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_down.7.weight",
"offset": 180283785216,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.ffn_up.7.weight",
"offset": 180485111808,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.35.attn_norm.weight",
"offset": 180686438400,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 180686462976,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.36.ffn_gate.0.weight",
"offset": 180686487552,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.36.ffn_down.0.weight",
"offset": 180887814144,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_up.0.weight",
"offset": 181089140736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate.1.weight",
"offset": 181290467328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_down.1.weight",
"offset": 181491793920,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_up.1.weight",
"offset": 181693120512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate.2.weight",
"offset": 181894447104,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_down.2.weight",
"offset": 182095773696,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_up.2.weight",
"offset": 182297100288,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate.3.weight",
"offset": 182498426880,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_down.3.weight",
"offset": 182699753472,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_up.3.weight",
"offset": 182901080064,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate.4.weight",
"offset": 183102406656,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate_inp.weight",
"offset": 183303733248,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.attn_k.weight",
"offset": 183303831552,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.36.attn_output.weight",
"offset": 183316414464,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.36.attn_q.weight",
"offset": 183391911936,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.36.attn_v.weight",
"offset": 183467409408,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.36.ffn_down.4.weight",
"offset": 183479992320,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.36.ffn_up.4.weight",
"offset": 183681318912,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate.5.weight",
"offset": 183882645504,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_down.5.weight",
"offset": 184083972096,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_up.5.weight",
"offset": 184285298688,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate.6.weight",
"offset": 184486625280,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_down.6.weight",
"offset": 184687951872,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_up.6.weight",
"offset": 184889278464,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_gate.7.weight",
"offset": 185090605056,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_down.7.weight",
"offset": 185291931648,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.ffn_up.7.weight",
"offset": 185493258240,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.36.attn_norm.weight",
"offset": 185694584832,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 185694609408,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.37.ffn_gate.0.weight",
"offset": 185694633984,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.37.ffn_down.0.weight",
"offset": 185895960576,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.0.weight",
"offset": 186097287168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_gate.1.weight",
"offset": 186298613760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_down.1.weight",
"offset": 186499940352,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.1.weight",
"offset": 186701266944,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_gate.2.weight",
"offset": 186902593536,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_down.2.weight",
"offset": 187103920128,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.2.weight",
"offset": 187305246720,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_gate.3.weight",
"offset": 187506573312,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_down.3.weight",
"offset": 187707899904,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.3.weight",
"offset": 187909226496,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_gate_inp.weight",
"offset": 188110553088,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.attn_k.weight",
"offset": 188110651392,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.37.attn_output.weight",
"offset": 188123234304,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.37.attn_q.weight",
"offset": 188198731776,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.37.attn_v.weight",
"offset": 188274229248,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.37.ffn_gate.4.weight",
"offset": 188286812160,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.37.ffn_down.4.weight",
"offset": 188488138752,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.4.weight",
"offset": 188689465344,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_gate.5.weight",
"offset": 188890791936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_down.5.weight",
"offset": 189092118528,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.5.weight",
"offset": 189293445120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_gate.6.weight",
"offset": 189494771712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_down.6.weight",
"offset": 189696098304,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.6.weight",
"offset": 189897424896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_gate.7.weight",
"offset": 190098751488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_down.7.weight",
"offset": 190300078080,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.ffn_up.7.weight",
"offset": 190501404672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.37.attn_norm.weight",
"offset": 190702731264,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 190702755840,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.38.ffn_gate.0.weight",
"offset": 190702780416,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.38.ffn_down.0.weight",
"offset": 190904107008,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_up.0.weight",
"offset": 191105433600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_gate.1.weight",
"offset": 191306760192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_down.1.weight",
"offset": 191508086784,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_up.1.weight",
"offset": 191709413376,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_gate.2.weight",
"offset": 191910739968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_down.2.weight",
"offset": 192112066560,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_up.2.weight",
"offset": 192313393152,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_gate.3.weight",
"offset": 192514719744,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_down.3.weight",
"offset": 192716046336,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_gate_inp.weight",
"offset": 192917372928,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.attn_k.weight",
"offset": 192917471232,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.38.attn_output.weight",
"offset": 192930054144,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.38.attn_q.weight",
"offset": 193005551616,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.38.attn_v.weight",
"offset": 193081049088,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.38.ffn_up.3.weight",
"offset": 193093632000,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.38.ffn_gate.4.weight",
"offset": 193294958592,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_down.4.weight",
"offset": 193496285184,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_up.4.weight",
"offset": 193697611776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_gate.5.weight",
"offset": 193898938368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_down.5.weight",
"offset": 194100264960,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_up.5.weight",
"offset": 194301591552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_gate.6.weight",
"offset": 194502918144,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_down.6.weight",
"offset": 194704244736,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_up.6.weight",
"offset": 194905571328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_gate.7.weight",
"offset": 195106897920,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_down.7.weight",
"offset": 195308224512,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.ffn_up.7.weight",
"offset": 195509551104,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.38.attn_norm.weight",
"offset": 195710877696,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 195710902272,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.39.ffn_gate.0.weight",
"offset": 195710926848,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.39.ffn_down.0.weight",
"offset": 195912253440,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_up.0.weight",
"offset": 196113580032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate.1.weight",
"offset": 196314906624,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_down.1.weight",
"offset": 196516233216,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_up.1.weight",
"offset": 196717559808,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate.2.weight",
"offset": 196918886400,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_down.2.weight",
"offset": 197120212992,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_up.2.weight",
"offset": 197321539584,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate.3.weight",
"offset": 197522866176,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate_inp.weight",
"offset": 197724192768,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.attn_k.weight",
"offset": 197724291072,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.39.attn_output.weight",
"offset": 197736873984,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.39.attn_q.weight",
"offset": 197812371456,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.39.attn_v.weight",
"offset": 197887868928,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.39.ffn_down.3.weight",
"offset": 197900451840,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.39.ffn_up.3.weight",
"offset": 198101778432,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate.4.weight",
"offset": 198303105024,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_down.4.weight",
"offset": 198504431616,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_up.4.weight",
"offset": 198705758208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate.5.weight",
"offset": 198907084800,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_down.5.weight",
"offset": 199108411392,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_up.5.weight",
"offset": 199309737984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate.6.weight",
"offset": 199511064576,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_down.6.weight",
"offset": 199712391168,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_up.6.weight",
"offset": 199913717760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_gate.7.weight",
"offset": 200115044352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_down.7.weight",
"offset": 200316370944,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.ffn_up.7.weight",
"offset": 200517697536,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.39.attn_norm.weight",
"offset": 200719024128,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 200719048704,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.40.ffn_gate.0.weight",
"offset": 200719073280,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.40.ffn_down.0.weight",
"offset": 200920399872,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.0.weight",
"offset": 201121726464,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_gate.1.weight",
"offset": 201323053056,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_down.1.weight",
"offset": 201524379648,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.1.weight",
"offset": 201725706240,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_gate.2.weight",
"offset": 201927032832,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_down.2.weight",
"offset": 202128359424,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.2.weight",
"offset": 202329686016,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_gate_inp.weight",
"offset": 202531012608,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.attn_k.weight",
"offset": 202531110912,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.40.attn_output.weight",
"offset": 202543693824,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.40.attn_q.weight",
"offset": 202619191296,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.40.attn_v.weight",
"offset": 202694688768,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.40.ffn_gate.3.weight",
"offset": 202707271680,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.40.ffn_down.3.weight",
"offset": 202908598272,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.3.weight",
"offset": 203109924864,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_gate.4.weight",
"offset": 203311251456,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_down.4.weight",
"offset": 203512578048,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.4.weight",
"offset": 203713904640,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_gate.5.weight",
"offset": 203915231232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_down.5.weight",
"offset": 204116557824,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.5.weight",
"offset": 204317884416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_gate.6.weight",
"offset": 204519211008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_down.6.weight",
"offset": 204720537600,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.6.weight",
"offset": 204921864192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_gate.7.weight",
"offset": 205123190784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_down.7.weight",
"offset": 205324517376,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.ffn_up.7.weight",
"offset": 205525843968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.40.attn_norm.weight",
"offset": 205727170560,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 205727195136,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.41.ffn_gate.0.weight",
"offset": 205727219712,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.41.ffn_down.0.weight",
"offset": 205928546304,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_up.0.weight",
"offset": 206129872896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_gate.1.weight",
"offset": 206331199488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_down.1.weight",
"offset": 206532526080,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_up.1.weight",
"offset": 206733852672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_gate.2.weight",
"offset": 206935179264,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_down.2.weight",
"offset": 207136505856,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_gate_inp.weight",
"offset": 207337832448,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.attn_k.weight",
"offset": 207337930752,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.41.attn_output.weight",
"offset": 207350513664,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.41.attn_q.weight",
"offset": 207426011136,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.41.attn_v.weight",
"offset": 207501508608,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.41.ffn_up.2.weight",
"offset": 207514091520,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.41.ffn_gate.3.weight",
"offset": 207715418112,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_down.3.weight",
"offset": 207916744704,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_up.3.weight",
"offset": 208118071296,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_gate.4.weight",
"offset": 208319397888,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_down.4.weight",
"offset": 208520724480,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_up.4.weight",
"offset": 208722051072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_gate.5.weight",
"offset": 208923377664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_down.5.weight",
"offset": 209124704256,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_up.5.weight",
"offset": 209326030848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_gate.6.weight",
"offset": 209527357440,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_down.6.weight",
"offset": 209728684032,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_up.6.weight",
"offset": 209930010624,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_gate.7.weight",
"offset": 210131337216,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_down.7.weight",
"offset": 210332663808,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.ffn_up.7.weight",
"offset": 210533990400,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.41.attn_norm.weight",
"offset": 210735316992,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 210735341568,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.42.ffn_gate.0.weight",
"offset": 210735366144,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.42.ffn_down.0.weight",
"offset": 210936692736,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_up.0.weight",
"offset": 211138019328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate.1.weight",
"offset": 211339345920,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_down.1.weight",
"offset": 211540672512,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_up.1.weight",
"offset": 211741999104,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate.2.weight",
"offset": 211943325696,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate_inp.weight",
"offset": 212144652288,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.attn_k.weight",
"offset": 212144750592,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.42.attn_output.weight",
"offset": 212157333504,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.42.attn_q.weight",
"offset": 212232830976,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.42.attn_v.weight",
"offset": 212308328448,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.42.ffn_down.2.weight",
"offset": 212320911360,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.42.ffn_up.2.weight",
"offset": 212522237952,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate.3.weight",
"offset": 212723564544,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_down.3.weight",
"offset": 212924891136,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_up.3.weight",
"offset": 213126217728,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate.4.weight",
"offset": 213327544320,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_down.4.weight",
"offset": 213528870912,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_up.4.weight",
"offset": 213730197504,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate.5.weight",
"offset": 213931524096,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_down.5.weight",
"offset": 214132850688,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_up.5.weight",
"offset": 214334177280,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate.6.weight",
"offset": 214535503872,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_down.6.weight",
"offset": 214736830464,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_up.6.weight",
"offset": 214938157056,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_gate.7.weight",
"offset": 215139483648,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_down.7.weight",
"offset": 215340810240,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.ffn_up.7.weight",
"offset": 215542136832,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.42.attn_norm.weight",
"offset": 215743463424,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 215743488000,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.43.ffn_gate.0.weight",
"offset": 215743512576,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.43.ffn_down.0.weight",
"offset": 215944839168,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.0.weight",
"offset": 216146165760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_gate.1.weight",
"offset": 216347492352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_down.1.weight",
"offset": 216548818944,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.1.weight",
"offset": 216750145536,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_gate_inp.weight",
"offset": 216951472128,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.attn_k.weight",
"offset": 216951570432,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.43.attn_output.weight",
"offset": 216964153344,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.43.attn_q.weight",
"offset": 217039650816,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.43.attn_v.weight",
"offset": 217115148288,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.43.ffn_gate.2.weight",
"offset": 217127731200,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.43.ffn_down.2.weight",
"offset": 217329057792,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.2.weight",
"offset": 217530384384,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_gate.3.weight",
"offset": 217731710976,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_down.3.weight",
"offset": 217933037568,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.3.weight",
"offset": 218134364160,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_gate.4.weight",
"offset": 218335690752,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_down.4.weight",
"offset": 218537017344,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.4.weight",
"offset": 218738343936,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_gate.5.weight",
"offset": 218939670528,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_down.5.weight",
"offset": 219140997120,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.5.weight",
"offset": 219342323712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_gate.6.weight",
"offset": 219543650304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_down.6.weight",
"offset": 219744976896,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.6.weight",
"offset": 219946303488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_gate.7.weight",
"offset": 220147630080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_down.7.weight",
"offset": 220348956672,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.ffn_up.7.weight",
"offset": 220550283264,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.43.attn_norm.weight",
"offset": 220751609856,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 220751634432,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.44.ffn_gate.0.weight",
"offset": 220751659008,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.44.ffn_down.0.weight",
"offset": 220952985600,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_up.0.weight",
"offset": 221154312192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_gate.1.weight",
"offset": 221355638784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_down.1.weight",
"offset": 221556965376,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_gate_inp.weight",
"offset": 221758291968,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.attn_k.weight",
"offset": 221758390272,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.44.attn_output.weight",
"offset": 221770973184,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.44.attn_q.weight",
"offset": 221846470656,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.44.attn_v.weight",
"offset": 221921968128,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.44.ffn_up.1.weight",
"offset": 221934551040,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.44.ffn_gate.2.weight",
"offset": 222135877632,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_down.2.weight",
"offset": 222337204224,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_up.2.weight",
"offset": 222538530816,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_gate.3.weight",
"offset": 222739857408,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_down.3.weight",
"offset": 222941184000,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_up.3.weight",
"offset": 223142510592,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_gate.4.weight",
"offset": 223343837184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_down.4.weight",
"offset": 223545163776,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_up.4.weight",
"offset": 223746490368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_gate.5.weight",
"offset": 223947816960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_down.5.weight",
"offset": 224149143552,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_up.5.weight",
"offset": 224350470144,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_gate.6.weight",
"offset": 224551796736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_down.6.weight",
"offset": 224753123328,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_up.6.weight",
"offset": 224954449920,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_gate.7.weight",
"offset": 225155776512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_down.7.weight",
"offset": 225357103104,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.ffn_up.7.weight",
"offset": 225558429696,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.44.attn_norm.weight",
"offset": 225759756288,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 225759780864,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.45.ffn_gate.0.weight",
"offset": 225759805440,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.45.ffn_down.0.weight",
"offset": 225961132032,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_up.0.weight",
"offset": 226162458624,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate.1.weight",
"offset": 226363785216,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate_inp.weight",
"offset": 226565111808,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.attn_k.weight",
"offset": 226565210112,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.45.attn_output.weight",
"offset": 226577793024,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.45.attn_q.weight",
"offset": 226653290496,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.45.attn_v.weight",
"offset": 226728787968,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.45.ffn_down.1.weight",
"offset": 226741370880,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.45.ffn_up.1.weight",
"offset": 226942697472,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate.2.weight",
"offset": 227144024064,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_down.2.weight",
"offset": 227345350656,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_up.2.weight",
"offset": 227546677248,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate.3.weight",
"offset": 227748003840,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_down.3.weight",
"offset": 227949330432,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_up.3.weight",
"offset": 228150657024,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate.4.weight",
"offset": 228351983616,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_down.4.weight",
"offset": 228553310208,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_up.4.weight",
"offset": 228754636800,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate.5.weight",
"offset": 228955963392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_down.5.weight",
"offset": 229157289984,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_up.5.weight",
"offset": 229358616576,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate.6.weight",
"offset": 229559943168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_down.6.weight",
"offset": 229761269760,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_up.6.weight",
"offset": 229962596352,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_gate.7.weight",
"offset": 230163922944,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_down.7.weight",
"offset": 230365249536,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.ffn_up.7.weight",
"offset": 230566576128,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.45.attn_norm.weight",
"offset": 230767902720,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 230767927296,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.46.ffn_gate.0.weight",
"offset": 230767951872,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.46.ffn_down.0.weight",
"offset": 230969278464,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.0.weight",
"offset": 231170605056,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_gate_inp.weight",
"offset": 231371931648,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.attn_k.weight",
"offset": 231372029952,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.46.attn_output.weight",
"offset": 231384612864,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.46.attn_q.weight",
"offset": 231460110336,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.46.attn_v.weight",
"offset": 231535607808,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.46.ffn_gate.1.weight",
"offset": 231548190720,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.46.ffn_down.1.weight",
"offset": 231749517312,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.1.weight",
"offset": 231950843904,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_gate.2.weight",
"offset": 232152170496,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_down.2.weight",
"offset": 232353497088,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.2.weight",
"offset": 232554823680,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_gate.3.weight",
"offset": 232756150272,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_down.3.weight",
"offset": 232957476864,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.3.weight",
"offset": 233158803456,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_gate.4.weight",
"offset": 233360130048,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_down.4.weight",
"offset": 233561456640,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.4.weight",
"offset": 233762783232,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_gate.5.weight",
"offset": 233964109824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_down.5.weight",
"offset": 234165436416,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.5.weight",
"offset": 234366763008,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_gate.6.weight",
"offset": 234568089600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_down.6.weight",
"offset": 234769416192,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.6.weight",
"offset": 234970742784,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_gate.7.weight",
"offset": 235172069376,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_down.7.weight",
"offset": 235373395968,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.ffn_up.7.weight",
"offset": 235574722560,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.46.attn_norm.weight",
"offset": 235776049152,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 235776073728,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.47.ffn_gate.0.weight",
"offset": 235776098304,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.47.ffn_down.0.weight",
"offset": 235977424896,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_gate_inp.weight",
"offset": 236178751488,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.attn_k.weight",
"offset": 236178849792,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.47.attn_output.weight",
"offset": 236191432704,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.47.attn_q.weight",
"offset": 236266930176,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.47.attn_v.weight",
"offset": 236342427648,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.47.ffn_up.0.weight",
"offset": 236355010560,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.47.ffn_gate.1.weight",
"offset": 236556337152,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_down.1.weight",
"offset": 236757663744,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_up.1.weight",
"offset": 236958990336,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_gate.2.weight",
"offset": 237160316928,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_down.2.weight",
"offset": 237361643520,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_up.2.weight",
"offset": 237562970112,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_gate.3.weight",
"offset": 237764296704,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_down.3.weight",
"offset": 237965623296,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_up.3.weight",
"offset": 238166949888,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_gate.4.weight",
"offset": 238368276480,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_down.4.weight",
"offset": 238569603072,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_up.4.weight",
"offset": 238770929664,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_gate.5.weight",
"offset": 238972256256,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_down.5.weight",
"offset": 239173582848,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_up.5.weight",
"offset": 239374909440,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_gate.6.weight",
"offset": 239576236032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_down.6.weight",
"offset": 239777562624,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_up.6.weight",
"offset": 239978889216,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_gate.7.weight",
"offset": 240180215808,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_down.7.weight",
"offset": 240381542400,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.ffn_up.7.weight",
"offset": 240582868992,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.47.attn_norm.weight",
"offset": 240784195584,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 240784220160,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.48.ffn_gate.0.weight",
"offset": 240784244736,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.48.ffn_gate_inp.weight",
"offset": 240985571328,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.attn_k.weight",
"offset": 240985669632,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.48.attn_output.weight",
"offset": 240998252544,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.48.attn_q.weight",
"offset": 241073750016,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.48.attn_v.weight",
"offset": 241149247488,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.48.ffn_down.0.weight",
"offset": 241161830400,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.48.ffn_up.0.weight",
"offset": 241363156992,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_gate.1.weight",
"offset": 241564483584,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_down.1.weight",
"offset": 241765810176,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_up.1.weight",
"offset": 241967136768,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_gate.2.weight",
"offset": 242168463360,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_down.2.weight",
"offset": 242369789952,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_up.2.weight",
"offset": 242571116544,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_gate.3.weight",
"offset": 242772443136,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_down.3.weight",
"offset": 242973769728,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_up.3.weight",
"offset": 243175096320,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_gate.4.weight",
"offset": 243376422912,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_down.4.weight",
"offset": 243577749504,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_up.4.weight",
"offset": 243779076096,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_gate.5.weight",
"offset": 243980402688,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_down.5.weight",
"offset": 244181729280,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_up.5.weight",
"offset": 244383055872,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_gate.6.weight",
"offset": 244584382464,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_down.6.weight",
"offset": 244785709056,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_up.6.weight",
"offset": 244987035648,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_gate.7.weight",
"offset": 245188362240,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_down.7.weight",
"offset": 245389688832,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.ffn_up.7.weight",
"offset": 245591015424,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.48.attn_norm.weight",
"offset": 245792342016,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 245792366592,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.49.ffn_gate_inp.weight",
"offset": 245792391168,
"shape": [
6144,
8
],
"size": 24576,
"type": 1
},
{
"name": "blk.49.attn_k.weight",
"offset": 245792489472,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.49.attn_output.weight",
"offset": 245805072384,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.49.attn_q.weight",
"offset": 245880569856,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.49.attn_v.weight",
"offset": 245956067328,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.49.ffn_gate.0.weight",
"offset": 245968650240,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.49.ffn_down.0.weight",
"offset": 246169976832,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.0.weight",
"offset": 246371303424,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_gate.1.weight",
"offset": 246572630016,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_down.1.weight",
"offset": 246773956608,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.1.weight",
"offset": 246975283200,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_gate.2.weight",
"offset": 247176609792,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_down.2.weight",
"offset": 247377936384,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.2.weight",
"offset": 247579262976,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_gate.3.weight",
"offset": 247780589568,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_down.3.weight",
"offset": 247981916160,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.3.weight",
"offset": 248183242752,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_gate.4.weight",
"offset": 248384569344,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_down.4.weight",
"offset": 248585895936,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.4.weight",
"offset": 248787222528,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_gate.5.weight",
"offset": 248988549120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_down.5.weight",
"offset": 249189875712,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.5.weight",
"offset": 249391202304,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_gate.6.weight",
"offset": 249592528896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_down.6.weight",
"offset": 249793855488,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.6.weight",
"offset": 249995182080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_gate.7.weight",
"offset": 250196508672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_down.7.weight",
"offset": 250397835264,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.ffn_up.7.weight",
"offset": 250599161856,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.49.attn_norm.weight",
"offset": 250800488448,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 250800513024,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 250800537600,
"shape": [
6144,
1024
],
"size": 24576,
"type": 1
},
{
"name": "blk.50.attn_q.weight",
"offset": 250813120512,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.50.attn_v.weight",
"offset": 250888617984,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.50.ffn_gate.0.weight",
"offset": 250901200896,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.50.ffn_down.0.weight",
"offset": 251102527488,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.0.weight",
"offset": 251303854080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate.1.weight",
"offset": 251505180672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_down.1.weight",
"offset": 251706507264,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.1.weight",
"offset": 251907833856,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate.2.weight",
"offset": 252109160448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_down.2.weight",
"offset": 252310487040,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.2.weight",
"offset": 252511813632,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate.3.weight",
"offset": 252713140224,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_down.3.weight",
"offset": 252914466816,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.3.weight",
"offset": 253115793408,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate.4.weight",
"offset": 253317120000,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_down.4.weight",
"offset": 253518446592,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.4.weight",
"offset": 253719773184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate.5.weight",
"offset": 253921099776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_down.5.weight",
"offset": 254122426368,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.5.weight",
"offset": 254323752960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate.6.weight",
"offset": 254525079552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_down.6.weight",
"offset": 254726406144,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.6.weight",
"offset": 254927732736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate.7.weight",
"offset": 255129059328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_down.7.weight",
"offset": 255330385920,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_up.7.weight",
"offset": 255531712512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.ffn_gate_inp.weight",
"offset": 255733039104,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.50.attn_norm.weight",
"offset": 255733137408,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 255733161984,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.50.attn_output.weight",
"offset": 255733186560,
"shape": [
6144,
6144
],
"size": 24576,
"type": 1
},
{
"name": "blk.51.attn_k.weight",
"offset": 255808684032,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.51.attn_q.weight",
"offset": 255821266944,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.51.ffn_gate.0.weight",
"offset": 255896764416,
"shape": [
6144,
16384
],
"size": 75497472,
"type": 1
},
{
"name": "blk.51.ffn_down.0.weight",
"offset": 256098091008,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.0.weight",
"offset": 256299417600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate.1.weight",
"offset": 256500744192,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_down.1.weight",
"offset": 256702070784,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.1.weight",
"offset": 256903397376,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate.2.weight",
"offset": 257104723968,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_down.2.weight",
"offset": 257306050560,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.2.weight",
"offset": 257507377152,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate.3.weight",
"offset": 257708703744,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_down.3.weight",
"offset": 257910030336,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.3.weight",
"offset": 258111356928,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate.4.weight",
"offset": 258312683520,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_down.4.weight",
"offset": 258514010112,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.4.weight",
"offset": 258715336704,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate.5.weight",
"offset": 258916663296,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_down.5.weight",
"offset": 259117989888,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.5.weight",
"offset": 259319316480,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate.6.weight",
"offset": 259520643072,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_down.6.weight",
"offset": 259721969664,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.6.weight",
"offset": 259923296256,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate.7.weight",
"offset": 260124622848,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_down.7.weight",
"offset": 260325949440,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_up.7.weight",
"offset": 260527276032,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.ffn_gate_inp.weight",
"offset": 260728602624,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.51.attn_norm.weight",
"offset": 260728700928,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 260728725504,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.51.attn_output.weight",
"offset": 260728750080,
"shape": [
6144,
6144
],
"size": 24576,
"type": 1
},
{
"name": "blk.51.attn_v.weight",
"offset": 260804247552,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.52.attn_q.weight",
"offset": 260816830464,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.52.ffn_gate.0.weight",
"offset": 260892327936,
"shape": [
6144,
16384
],
"size": 75497472,
"type": 1
},
{
"name": "blk.52.ffn_down.0.weight",
"offset": 261093654528,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.0.weight",
"offset": 261294981120,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate.1.weight",
"offset": 261496307712,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_down.1.weight",
"offset": 261697634304,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.1.weight",
"offset": 261898960896,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate.2.weight",
"offset": 262100287488,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_down.2.weight",
"offset": 262301614080,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.2.weight",
"offset": 262502940672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate.3.weight",
"offset": 262704267264,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_down.3.weight",
"offset": 262905593856,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.3.weight",
"offset": 263106920448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate.4.weight",
"offset": 263308247040,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_down.4.weight",
"offset": 263509573632,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.4.weight",
"offset": 263710900224,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate.5.weight",
"offset": 263912226816,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_down.5.weight",
"offset": 264113553408,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.5.weight",
"offset": 264314880000,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate.6.weight",
"offset": 264516206592,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_down.6.weight",
"offset": 264717533184,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.6.weight",
"offset": 264918859776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate.7.weight",
"offset": 265120186368,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_down.7.weight",
"offset": 265321512960,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_up.7.weight",
"offset": 265522839552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.ffn_gate_inp.weight",
"offset": 265724166144,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.52.attn_norm.weight",
"offset": 265724264448,
"shape": [
6144
],
"size": 98304,
"type": 0
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 265724289024,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 265724313600,
"shape": [
6144,
1024
],
"size": 24576,
"type": 1
},
{
"name": "blk.52.attn_output.weight",
"offset": 265736896512,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.52.attn_v.weight",
"offset": 265812393984,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.53.ffn_gate.0.weight",
"offset": 265824976896,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.53.ffn_down.0.weight",
"offset": 266026303488,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_up.0.weight",
"offset": 266227630080,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate.1.weight",
"offset": 266428956672,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_down.1.weight",
"offset": 266630283264,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_up.1.weight",
"offset": 266831609856,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate.2.weight",
"offset": 267032936448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_down.2.weight",
"offset": 267234263040,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_up.2.weight",
"offset": 267435589632,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate.3.weight",
"offset": 267636916224,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_down.3.weight",
"offset": 267838242816,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_up.3.weight",
"offset": 268039569408,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate.4.weight",
"offset": 268240896000,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_down.4.weight",
"offset": 268442222592,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_up.4.weight",
"offset": 268643549184,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate.5.weight",
"offset": 268844875776,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_down.5.weight",
"offset": 269046202368,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_up.5.weight",
"offset": 269247528960,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate.6.weight",
"offset": 269448855552,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_down.6.weight",
"offset": 269650182144,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_up.6.weight",
"offset": 269851508736,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate.7.weight",
"offset": 270052835328,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_down.7.weight",
"offset": 270254161920,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.ffn_gate_inp.weight",
"offset": 270455488512,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.53.attn_k.weight",
"offset": 270455586816,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.53.attn_output.weight",
"offset": 270468169728,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.53.attn_q.weight",
"offset": 270543667200,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.53.attn_v.weight",
"offset": 270619164672,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.53.ffn_up.7.weight",
"offset": 270631747584,
"shape": [
6144,
16384
],
"size": 12582912,
"type": 1
},
{
"name": "blk.53.attn_norm.weight",
"offset": 270833074176,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 270833098752,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.54.ffn_gate.0.weight",
"offset": 270833123328,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.54.ffn_down.0.weight",
"offset": 271034449920,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_up.0.weight",
"offset": 271235776512,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate.1.weight",
"offset": 271437103104,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_down.1.weight",
"offset": 271638429696,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_up.1.weight",
"offset": 271839756288,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate.2.weight",
"offset": 272041082880,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_down.2.weight",
"offset": 272242409472,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_up.2.weight",
"offset": 272443736064,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate.3.weight",
"offset": 272645062656,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_down.3.weight",
"offset": 272846389248,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_up.3.weight",
"offset": 273047715840,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate.4.weight",
"offset": 273249042432,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_down.4.weight",
"offset": 273450369024,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_up.4.weight",
"offset": 273651695616,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate.5.weight",
"offset": 273853022208,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_down.5.weight",
"offset": 274054348800,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_up.5.weight",
"offset": 274255675392,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate.6.weight",
"offset": 274457001984,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_down.6.weight",
"offset": 274658328576,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_up.6.weight",
"offset": 274859655168,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate.7.weight",
"offset": 275060981760,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.ffn_gate_inp.weight",
"offset": 275262308352,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.attn_k.weight",
"offset": 275262406656,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.54.attn_output.weight",
"offset": 275274989568,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.54.attn_q.weight",
"offset": 275350487040,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.54.attn_v.weight",
"offset": 275425984512,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "blk.54.ffn_down.7.weight",
"offset": 275438567424,
"shape": [
16384,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.54.ffn_up.7.weight",
"offset": 275639894016,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.54.attn_norm.weight",
"offset": 275841220608,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 275841245184,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "blk.55.ffn_gate.0.weight",
"offset": 275841269760,
"shape": [
6144,
16384
],
"size": 24576,
"type": 1
},
{
"name": "blk.55.ffn_down.0.weight",
"offset": 276042596352,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.0.weight",
"offset": 276243922944,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_gate.1.weight",
"offset": 276445249536,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_down.1.weight",
"offset": 276646576128,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.1.weight",
"offset": 276847902720,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_gate.2.weight",
"offset": 277049229312,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_down.2.weight",
"offset": 277250555904,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.2.weight",
"offset": 277451882496,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_gate.3.weight",
"offset": 277653209088,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_down.3.weight",
"offset": 277854535680,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.3.weight",
"offset": 278055862272,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_gate.4.weight",
"offset": 278257188864,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_down.4.weight",
"offset": 278458515456,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.4.weight",
"offset": 278659842048,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_gate.5.weight",
"offset": 278861168640,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_down.5.weight",
"offset": 279062495232,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.5.weight",
"offset": 279263821824,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_gate.6.weight",
"offset": 279465148416,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_down.6.weight",
"offset": 279666475008,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.6.weight",
"offset": 279867801600,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_gate_inp.weight",
"offset": 280069128192,
"shape": [
6144,
8
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.attn_k.weight",
"offset": 280069226496,
"shape": [
6144,
1024
],
"size": 98304,
"type": 1
},
{
"name": "blk.55.attn_output.weight",
"offset": 280081809408,
"shape": [
6144,
6144
],
"size": 12582912,
"type": 1
},
{
"name": "blk.55.attn_q.weight",
"offset": 280157306880,
"shape": [
6144,
6144
],
"size": 75497472,
"type": 1
},
{
"name": "blk.55.attn_v.weight",
"offset": 280232804352,
"shape": [
6144,
1024
],
"size": 75497472,
"type": 1
},
{
"name": "output.weight",
"offset": 280245387264,
"shape": [
6144,
32000
],
"size": 12582912,
"type": 1
},
{
"name": "blk.55.ffn_gate.7.weight",
"offset": 280638603264,
"shape": [
6144,
16384
],
"size": 393216000,
"type": 1
},
{
"name": "blk.55.ffn_down.7.weight",
"offset": 280839929856,
"shape": [
16384,
6144
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.ffn_up.7.weight",
"offset": 281041256448,
"shape": [
6144,
16384
],
"size": 201326592,
"type": 1
},
{
"name": "blk.55.attn_norm.weight",
"offset": 281242583040,
"shape": [
6144
],
"size": 201326592,
"type": 0
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 281242607616,
"shape": [
6144
],
"size": 24576,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 281242632192,
"shape": [
6144
],
"size": 24576,
"type": 0
}
],
"version": 3
}