Qwen2 MOE 57B
298 Pulls Updated 3 weeks ago
5c9d9bae8e97 · 41GB
{
"metadata": {
"general.architecture": "qwen2moe",
"general.file_type": 17,
"general.name": "Qwen2-57B-A14B-Instruct",
"general.quantization_version": 2,
"general.source.url": "https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct",
"general.url": "https://huggingface.co/mradermacher/Qwen2-57B-A14B-Instruct-GGUF",
"mradermacher.convert_type": "hf",
"mradermacher.quantize_version": "2",
"mradermacher.quantized_at": "2024-06-07T17:06:12+02:00",
"mradermacher.quantized_by": "mradermacher",
"mradermacher.quantized_on": "kaos",
"qwen2moe.attention.head_count": 28,
"qwen2moe.attention.head_count_kv": 4,
"qwen2moe.attention.layer_norm_rms_epsilon": 0.000001,
"qwen2moe.block_count": 28,
"qwen2moe.context_length": 32768,
"qwen2moe.embedding_length": 3584,
"qwen2moe.expert_count": 64,
"qwen2moe.expert_used_count": 8,
"qwen2moe.feed_forward_length": 20480,
"qwen2moe.rope.freq_base": 1000000,
"tokenizer.ggml.bos_token_id": 151643,
"tokenizer.ggml.eos_token_id": 151645,
"tokenizer.ggml.merges": "... (151387 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 151643,
"tokenizer.ggml.pre": "qwen2",
"tokenizer.ggml.token_type": "... (151936 values)",
"tokenizer.ggml.tokens": "... (151936 values)"
},
"num_params": 57408658944,
"tensors": [
{
"name": "blk.0.ffn_gate_inp.weight",
"offset": 374370304,
"shape": [
3584,
64
],
"size": 374370304,
"type": 0
},
{
"name": "blk.0.attn_k.bias",
"offset": 375287808,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 375289856,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 376551424,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.0.attn_q.bias",
"offset": 385382400,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.0.attn_q.weight",
"offset": 385396736,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.0.attn_v.bias",
"offset": 394227712,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.0.attn_v.weight",
"offset": 394229760,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.0.attn_norm.weight",
"offset": 395735040,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.0.ffn_down_exps.weight",
"offset": 395749376,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.0.ffn_gate_exps.weight",
"offset": 877438976,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.0.ffn_up_exps.weight",
"offset": 1281140736,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.0.ffn_down_shexp.weight",
"offset": 1684842496,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.0.ffn_gate_shexp.weight",
"offset": 1745053696,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.0.ffn_up_shexp.weight",
"offset": 1795516416,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.0.ffn_gate_inp_shexp.weight",
"offset": 1845979136,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1845993472,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.ffn_gate_inp.weight",
"offset": 1846007808,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.attn_k.bias",
"offset": 1846925312,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1846927360,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 1848188928,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.1.attn_q.bias",
"offset": 1857019904,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 1857034240,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.1.attn_v.bias",
"offset": 1865865216,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.1.attn_v.weight",
"offset": 1865867264,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 1867372544,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.1.ffn_down_exps.weight",
"offset": 1867386880,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.1.ffn_gate_exps.weight",
"offset": 2349076480,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.1.ffn_up_exps.weight",
"offset": 2752778240,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.1.ffn_down_shexp.weight",
"offset": 3156480000,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.1.ffn_gate_shexp.weight",
"offset": 3216691200,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.1.ffn_up_shexp.weight",
"offset": 3267153920,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.1.ffn_gate_inp_shexp.weight",
"offset": 3317616640,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 3317630976,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.ffn_gate_inp.weight",
"offset": 3317645312,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.attn_k.bias",
"offset": 3318562816,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 3318564864,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 3319826432,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.2.attn_q.bias",
"offset": 3328657408,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 3328671744,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.2.attn_v.bias",
"offset": 3337502720,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.2.attn_v.weight",
"offset": 3337504768,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 3339010048,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.2.ffn_down_exps.weight",
"offset": 3339024384,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.2.ffn_gate_exps.weight",
"offset": 3820713984,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.2.ffn_up_exps.weight",
"offset": 4224415744,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.2.ffn_down_shexp.weight",
"offset": 4628117504,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.2.ffn_gate_shexp.weight",
"offset": 4688328704,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.2.ffn_up_shexp.weight",
"offset": 4738791424,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.2.ffn_gate_inp_shexp.weight",
"offset": 4789254144,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 4789268480,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.ffn_gate_inp.weight",
"offset": 4789282816,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.attn_k.bias",
"offset": 4790200320,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 4790202368,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 4791463936,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.3.attn_q.bias",
"offset": 4800294912,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 4800309248,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.3.attn_v.bias",
"offset": 4809140224,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.3.attn_v.weight",
"offset": 4809142272,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.3.attn_norm.weight",
"offset": 4810403840,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.3.ffn_down_exps.weight",
"offset": 4810418176,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.3.ffn_gate_exps.weight",
"offset": 5214119936,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.3.ffn_up_exps.weight",
"offset": 5617821696,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.3.ffn_down_shexp.weight",
"offset": 6021523456,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.3.ffn_gate_shexp.weight",
"offset": 6071986176,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.3.ffn_up_shexp.weight",
"offset": 6122448896,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.3.ffn_gate_inp_shexp.weight",
"offset": 6172911616,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 6172925952,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.ffn_gate_inp.weight",
"offset": 6172940288,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.attn_k.bias",
"offset": 6173857792,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 6173859840,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 6175121408,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.4.attn_q.bias",
"offset": 6183952384,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 6183966720,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.4.attn_v.bias",
"offset": 6192797696,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.4.attn_v.weight",
"offset": 6192799744,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.4.attn_norm.weight",
"offset": 6194061312,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.4.ffn_down_exps.weight",
"offset": 6194075648,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.4.ffn_gate_exps.weight",
"offset": 6597777408,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.4.ffn_up_exps.weight",
"offset": 7001479168,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.4.ffn_down_shexp.weight",
"offset": 7405180928,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.4.ffn_gate_shexp.weight",
"offset": 7455643648,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.4.ffn_up_shexp.weight",
"offset": 7506106368,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.4.ffn_gate_inp_shexp.weight",
"offset": 7556569088,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 7556583424,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.ffn_gate_inp.weight",
"offset": 7556597760,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.attn_k.bias",
"offset": 7557515264,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 7557517312,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 7558778880,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.5.attn_q.bias",
"offset": 7567609856,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 7567624192,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.5.attn_v.bias",
"offset": 7576455168,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.5.attn_v.weight",
"offset": 7576457216,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 7577962496,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.5.ffn_down_exps.weight",
"offset": 7577976832,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.5.ffn_gate_exps.weight",
"offset": 8059666432,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.5.ffn_up_exps.weight",
"offset": 8463368192,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.5.ffn_down_shexp.weight",
"offset": 8867069952,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.5.ffn_gate_shexp.weight",
"offset": 8927281152,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.5.ffn_up_shexp.weight",
"offset": 8977743872,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.5.ffn_gate_inp_shexp.weight",
"offset": 9028206592,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 9028220928,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.ffn_gate_inp.weight",
"offset": 9028235264,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.attn_k.bias",
"offset": 9029152768,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 9029154816,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 9030416384,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.6.attn_q.bias",
"offset": 9039247360,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 9039261696,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.6.attn_v.bias",
"offset": 9048092672,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.6.attn_v.weight",
"offset": 9048094720,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.6.attn_norm.weight",
"offset": 9049356288,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.6.ffn_down_exps.weight",
"offset": 9049370624,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.6.ffn_gate_exps.weight",
"offset": 9453072384,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.6.ffn_up_exps.weight",
"offset": 9856774144,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.6.ffn_down_shexp.weight",
"offset": 10260475904,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.6.ffn_gate_shexp.weight",
"offset": 10310938624,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.6.ffn_up_shexp.weight",
"offset": 10361401344,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.6.ffn_gate_inp_shexp.weight",
"offset": 10411864064,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 10411878400,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.ffn_gate_inp.weight",
"offset": 10411892736,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.attn_k.bias",
"offset": 10412810240,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 10412812288,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 10414073856,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.7.attn_q.bias",
"offset": 10422904832,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 10422919168,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.7.attn_v.bias",
"offset": 10431750144,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.7.attn_v.weight",
"offset": 10431752192,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.7.attn_norm.weight",
"offset": 10433013760,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.7.ffn_down_exps.weight",
"offset": 10433028096,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.7.ffn_gate_exps.weight",
"offset": 10836729856,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.7.ffn_up_exps.weight",
"offset": 11240431616,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.7.ffn_down_shexp.weight",
"offset": 11644133376,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.7.ffn_gate_shexp.weight",
"offset": 11694596096,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.7.ffn_up_shexp.weight",
"offset": 11745058816,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.7.ffn_gate_inp_shexp.weight",
"offset": 11795521536,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 11795535872,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.ffn_gate_inp.weight",
"offset": 11795550208,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.attn_k.bias",
"offset": 11796467712,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 11796469760,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 11797731328,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.8.attn_q.bias",
"offset": 11806562304,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 11806576640,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.8.attn_v.bias",
"offset": 11815407616,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.8.attn_v.weight",
"offset": 11815409664,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 11816914944,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.8.ffn_down_exps.weight",
"offset": 11816929280,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.8.ffn_gate_exps.weight",
"offset": 12298618880,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.8.ffn_up_exps.weight",
"offset": 12702320640,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.8.ffn_down_shexp.weight",
"offset": 13106022400,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.8.ffn_gate_shexp.weight",
"offset": 13166233600,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.8.ffn_up_shexp.weight",
"offset": 13216696320,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.8.ffn_gate_inp_shexp.weight",
"offset": 13267159040,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 13267173376,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.ffn_gate_inp.weight",
"offset": 13267187712,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.attn_k.bias",
"offset": 13268105216,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 13268107264,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 13269368832,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.9.attn_q.bias",
"offset": 13278199808,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 13278214144,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.9.attn_v.bias",
"offset": 13287045120,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.9.attn_v.weight",
"offset": 13287047168,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.10.ffn_gate_inp.weight",
"offset": 13288308736,
"shape": [
3584,
64
],
"size": 1261568,
"type": 0
},
{
"name": "blk.10.attn_k.bias",
"offset": 13289226240,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 13289228288,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 13290489856,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.10.attn_q.bias",
"offset": 13299320832,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 13299335168,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.10.attn_v.bias",
"offset": 13308166144,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.10.attn_v.weight",
"offset": 13308168192,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.9.attn_norm.weight",
"offset": 13309429760,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.9.ffn_down_exps.weight",
"offset": 13309444096,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.9.ffn_gate_exps.weight",
"offset": 13713145856,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.9.ffn_up_exps.weight",
"offset": 14116847616,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.9.ffn_down_shexp.weight",
"offset": 14520549376,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.9.ffn_gate_shexp.weight",
"offset": 14571012096,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.9.ffn_up_shexp.weight",
"offset": 14621474816,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.9.ffn_gate_inp_shexp.weight",
"offset": 14671937536,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 14671951872,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.attn_norm.weight",
"offset": 14671966208,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.ffn_down_exps.weight",
"offset": 14671980544,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.10.ffn_gate_exps.weight",
"offset": 15075682304,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.10.ffn_up_exps.weight",
"offset": 15479384064,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.10.ffn_down_shexp.weight",
"offset": 15883085824,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.10.ffn_gate_shexp.weight",
"offset": 15933548544,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.10.ffn_up_shexp.weight",
"offset": 15984011264,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.10.ffn_gate_inp_shexp.weight",
"offset": 16034473984,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 16034488320,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.ffn_gate_inp.weight",
"offset": 16034502656,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.attn_k.bias",
"offset": 16035420160,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 16035422208,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 16036683776,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.11.attn_q.bias",
"offset": 16045514752,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 16045529088,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.11.attn_v.bias",
"offset": 16054360064,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.11.attn_v.weight",
"offset": 16054362112,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 16055867392,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.11.ffn_down_exps.weight",
"offset": 16055881728,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.11.ffn_gate_exps.weight",
"offset": 16537571328,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.11.ffn_up_exps.weight",
"offset": 16941273088,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.11.ffn_down_shexp.weight",
"offset": 17344974848,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.11.ffn_gate_shexp.weight",
"offset": 17405186048,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.11.ffn_up_shexp.weight",
"offset": 17455648768,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.11.ffn_gate_inp_shexp.weight",
"offset": 17506111488,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 17506125824,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.ffn_gate_inp.weight",
"offset": 17506140160,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.attn_k.bias",
"offset": 17507057664,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 17507059712,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 17508321280,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.12.attn_q.bias",
"offset": 17517152256,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 17517166592,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.12.attn_v.bias",
"offset": 17525997568,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.12.attn_v.weight",
"offset": 17525999616,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.12.attn_norm.weight",
"offset": 17527261184,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.12.ffn_down_exps.weight",
"offset": 17527275520,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.12.ffn_gate_exps.weight",
"offset": 17930977280,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.12.ffn_up_exps.weight",
"offset": 18334679040,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.12.ffn_down_shexp.weight",
"offset": 18738380800,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.12.ffn_gate_shexp.weight",
"offset": 18788843520,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.12.ffn_up_shexp.weight",
"offset": 18839306240,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.12.ffn_gate_inp_shexp.weight",
"offset": 18889768960,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 18889783296,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.ffn_gate_inp.weight",
"offset": 18889797632,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.attn_k.bias",
"offset": 18890715136,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 18890717184,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 18891978752,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.13.attn_q.bias",
"offset": 18900809728,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "_gate_inp.weight\u0002\u0000\u0000\u0000",
"offset": 18900824064,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.13.attn_v.bias",
"offset": 18909655040,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.13.attn_v.weight",
"offset": 18909657088,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.13.attn_norm.weight",
"offset": 18910918656,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.13.ffn_down_exps.weight",
"offset": 18910932992,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.13.ffn_gate_exps.weight",
"offset": 19314634752,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.13.ffn_up_exps.weight",
"offset": 19718336512,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.13.ffn_down_shexp.weight",
"offset": 20122038272,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.13.ffn_gate_shexp.weight",
"offset": 20172500992,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.13.ffn_up_shexp.weight",
"offset": 20222963712,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.13.ffn_gate_inp_shexp.weight",
"offset": 20273426432,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 20273440768,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.ffn_gate_inp.weight",
"offset": 20273455104,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.attn_k.bias",
"offset": 20274372608,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 20274374656,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 20275636224,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.14.attn_q.bias",
"offset": 20284467200,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 20284481536,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.14.attn_v.bias",
"offset": 20293312512,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.14.attn_v.weight",
"offset": 20293314560,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 20294819840,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.14.ffn_down_exps.weight",
"offset": 20294834176,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.14.ffn_gate_exps.weight",
"offset": 20776523776,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.14.ffn_up_exps.weight",
"offset": 21180225536,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.14.ffn_down_shexp.weight",
"offset": 21583927296,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.14.ffn_gate_shexp.weight",
"offset": 21644138496,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.14.ffn_up_shexp.weight",
"offset": 21694601216,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.14.ffn_gate_inp_shexp.weight",
"offset": 21745063936,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 21745078272,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.ffn_gate_inp.weight",
"offset": 21745092608,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.attn_k.bias",
"offset": 21746010112,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 21746012160,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 21747273728,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.15.attn_q.bias",
"offset": 21756104704,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 21756119040,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.15.attn_v.bias",
"offset": 21764950016,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.15.attn_v.weight",
"offset": 21764952064,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.15.attn_norm.weight",
"offset": 21766213632,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.15.ffn_down_exps.weight",
"offset": 21766227968,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.15.ffn_gate_exps.weight",
"offset": 22169929728,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.15.ffn_up_exps.weight",
"offset": 22573631488,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.15.ffn_down_shexp.weight",
"offset": 22977333248,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.15.ffn_gate_shexp.weight",
"offset": 23027795968,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.15.ffn_up_shexp.weight",
"offset": 23078258688,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.15.ffn_gate_inp_shexp.weight",
"offset": 23128721408,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 23128735744,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.ffn_gate_inp.weight",
"offset": 23128750080,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.attn_k.bias",
"offset": 23129667584,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 23129669632,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 23130931200,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.16.attn_q.bias",
"offset": 23139762176,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 23139776512,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.16.attn_v.bias",
"offset": 23148607488,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.16.attn_v.weight",
"offset": 23148609536,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.16.attn_norm.weight",
"offset": 23149871104,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.16.ffn_down_exps.weight",
"offset": 23149885440,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.16.ffn_gate_exps.weight",
"offset": 23553587200,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.16.ffn_up_exps.weight",
"offset": 23957288960,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.16.ffn_down_shexp.weight",
"offset": 24360990720,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.16.ffn_gate_shexp.weight",
"offset": 24411453440,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.16.ffn_up_shexp.weight",
"offset": 24461916160,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.16.ffn_gate_inp_shexp.weight",
"offset": 24512378880,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 24512393216,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.ffn_gate_inp.weight",
"offset": 24512407552,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.attn_k.bias",
"offset": 24513325056,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 24513327104,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 24514588672,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.17.attn_q.bias",
"offset": 24523419648,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 24523433984,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.17.attn_v.bias",
"offset": 24532264960,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.17.attn_v.weight",
"offset": 24532267008,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 24533772288,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.17.ffn_down_exps.weight",
"offset": 24533786624,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.17.ffn_gate_exps.weight",
"offset": 25015476224,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.17.ffn_up_exps.weight",
"offset": 25419177984,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.17.ffn_down_shexp.weight",
"offset": 25822879744,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.17.ffn_gate_shexp.weight",
"offset": 25883090944,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.17.ffn_up_shexp.weight",
"offset": 25933553664,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.17.ffn_gate_inp_shexp.weight",
"offset": 25984016384,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 25984030720,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.ffn_gate_inp.weight",
"offset": 25984045056,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.attn_k.bias",
"offset": 25984962560,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 25984964608,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 25986226176,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.18.attn_q.bias",
"offset": 25995057152,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 25995071488,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.18.attn_v.bias",
"offset": 26003902464,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.18.attn_v.weight",
"offset": 26003904512,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.18.attn_norm.weight",
"offset": 26005166080,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.18.ffn_down_exps.weight",
"offset": 26005180416,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.18.ffn_gate_exps.weight",
"offset": 26408882176,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.18.ffn_up_exps.weight",
"offset": 26812583936,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.18.ffn_down_shexp.weight",
"offset": 27216285696,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.18.ffn_gate_shexp.weight",
"offset": 27266748416,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.18.ffn_up_shexp.weight",
"offset": 27317211136,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.18.ffn_gate_inp_shexp.weight",
"offset": 27367673856,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 27367688192,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.ffn_gate_inp.weight",
"offset": 27367702528,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.attn_k.bias",
"offset": 27368620032,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 27368622080,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 27369883648,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.19.attn_q.bias",
"offset": 27378714624,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 27378728960,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.19.attn_v.bias",
"offset": 27387559936,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.19.attn_v.weight",
"offset": 27387561984,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.19.attn_norm.weight",
"offset": 27388823552,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.19.ffn_down_exps.weight",
"offset": 27388837888,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.19.ffn_gate_exps.weight",
"offset": 27792539648,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.19.ffn_up_exps.weight",
"offset": 28196241408,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.19.ffn_down_shexp.weight",
"offset": 28599943168,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.19.ffn_gate_shexp.weight",
"offset": 28650405888,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.19.ffn_up_shexp.weight",
"offset": 28700868608,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.19.ffn_gate_inp_shexp.weight",
"offset": 28751331328,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 28751345664,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.ffn_gate_inp.weight",
"offset": 28751360000,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.attn_k.bias",
"offset": 28752277504,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 28752279552,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 28753541120,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.20.attn_q.bias",
"offset": 28762372096,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 28762386432,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.20.attn_v.bias",
"offset": 28771217408,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.20.attn_v.weight",
"offset": 28771219456,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 28772724736,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.20.ffn_down_exps.weight",
"offset": 28772739072,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.20.ffn_gate_exps.weight",
"offset": 29254428672,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.20.ffn_up_exps.weight",
"offset": 29658130432,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.20.ffn_down_shexp.weight",
"offset": 30061832192,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.20.ffn_gate_shexp.weight",
"offset": 30122043392,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.20.ffn_up_shexp.weight",
"offset": 30172506112,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.20.ffn_gate_inp_shexp.weight",
"offset": 30222968832,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 30222983168,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.21.ffn_gate_inp.weight",
"offset": 30222997504,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.21.attn_k.bias",
"offset": 30223915008,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 30223917056,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 30225178624,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.21.attn_q.bias",
"offset": 30234009600,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 30234023936,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.21.attn_v.bias",
"offset": 30242854912,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.21.attn_v.weight",
"offset": 30242856960,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.21.attn_norm.weight",
"offset": 30244118528,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.21.ffn_down_exps.weight",
"offset": 30244132864,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.21.ffn_gate_exps.weight",
"offset": 30647834624,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.21.ffn_up_exps.weight",
"offset": 31051536384,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.21.ffn_down_shexp.weight",
"offset": 31455238144,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.21.ffn_gate_shexp.weight",
"offset": 31505700864,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.21.ffn_up_shexp.weight",
"offset": 31556163584,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.21.ffn_gate_inp_shexp.weight",
"offset": 31606626304,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 31606640640,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.ffn_gate_inp.weight",
"offset": 31606654976,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.attn_k.bias",
"offset": 31607572480,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 31607574528,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 31608836096,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.22.attn_q.bias",
"offset": 31617667072,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 31617681408,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.22.attn_v.bias",
"offset": 31626512384,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.22.attn_v.weight",
"offset": 31626514432,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.22.attn_norm.weight",
"offset": 31627776000,
"shape": [
3584
],
"size": 1261568,
"type": 0
},
{
"name": "blk.22.ffn_down_exps.weight",
"offset": 31627790336,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 13
},
{
"name": "blk.22.ffn_gate_exps.weight",
"offset": 32031492096,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.22.ffn_up_exps.weight",
"offset": 32435193856,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.22.ffn_down_shexp.weight",
"offset": 32838895616,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 13
},
{
"name": "blk.22.ffn_gate_shexp.weight",
"offset": 32889358336,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.22.ffn_up_shexp.weight",
"offset": 32939821056,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.22.ffn_gate_inp_shexp.weight",
"offset": 32990283776,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 32990298112,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.ffn_gate_inp.weight",
"offset": 32990312448,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.attn_k.bias",
"offset": 32991229952,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 32991232000,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 32992493568,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.23.attn_q.bias",
"offset": 33001324544,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 33001338880,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.23.attn_v.bias",
"offset": 33010169856,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.23.attn_v.weight",
"offset": 33010171904,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 33011677184,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.23.ffn_down_exps.weight",
"offset": 33011691520,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.23.ffn_gate_exps.weight",
"offset": 33493381120,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.23.ffn_up_exps.weight",
"offset": 33897082880,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.23.ffn_down_shexp.weight",
"offset": 34300784640,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.23.ffn_gate_shexp.weight",
"offset": 34360995840,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.23.ffn_up_shexp.weight",
"offset": 34411458560,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.23.ffn_gate_inp_shexp.weight",
"offset": 34461921280,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 34461935616,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.ffn_gate_inp.weight",
"offset": 34461949952,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.attn_k.bias",
"offset": 34462867456,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 34462869504,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 34464131072,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.24.attn_q.bias",
"offset": 34472962048,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 34472976384,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.24.attn_v.bias",
"offset": 34481807360,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.24.attn_v.weight",
"offset": 34481809408,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 34483314688,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.24.ffn_down_exps.weight",
"offset": 34483329024,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.24.ffn_gate_exps.weight",
"offset": 34965018624,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.24.ffn_up_exps.weight",
"offset": 35368720384,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.24.ffn_down_shexp.weight",
"offset": 35772422144,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.24.ffn_gate_shexp.weight",
"offset": 35832633344,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.24.ffn_up_shexp.weight",
"offset": 35883096064,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.24.ffn_gate_inp_shexp.weight",
"offset": 35933558784,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 35933573120,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.ffn_gate_inp.weight",
"offset": 35933587456,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.attn_k.bias",
"offset": 35934504960,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 35934507008,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 35935768576,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.25.attn_q.bias",
"offset": 35944599552,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 35944613888,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.25.attn_v.bias",
"offset": 35953444864,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.25.attn_v.weight",
"offset": 35953446912,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 35954952192,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.25.ffn_down_exps.weight",
"offset": 35954966528,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.25.ffn_gate_exps.weight",
"offset": 36436656128,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.25.ffn_up_exps.weight",
"offset": 36840357888,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.25.ffn_down_shexp.weight",
"offset": 37244059648,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.25.ffn_gate_shexp.weight",
"offset": 37304270848,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.25.ffn_up_shexp.weight",
"offset": 37354733568,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.25.ffn_gate_inp_shexp.weight",
"offset": 37405196288,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 37405210624,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.ffn_gate_inp.weight",
"offset": 37405224960,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.attn_k.bias",
"offset": 37406142464,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 37406144512,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 37407406080,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.26.attn_q.bias",
"offset": 37416237056,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 37416251392,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.26.attn_v.bias",
"offset": 37425082368,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.26.attn_v.weight",
"offset": 37425084416,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 37426589696,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.26.ffn_down_exps.weight",
"offset": 37426604032,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.26.ffn_gate_exps.weight",
"offset": 37908293632,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.26.ffn_up_exps.weight",
"offset": 38311995392,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.26.ffn_down_shexp.weight",
"offset": 38715697152,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.26.ffn_gate_shexp.weight",
"offset": 38775908352,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.26.ffn_up_shexp.weight",
"offset": 38826371072,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.26.ffn_gate_inp_shexp.weight",
"offset": 38876833792,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 38876848128,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.ffn_gate_inp.weight",
"offset": 38876862464,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.attn_k.bias",
"offset": 38877779968,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 38877782016,
"shape": [
3584,
512
],
"size": 2048,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 38879043584,
"shape": [
3584,
3584
],
"size": 1261568,
"type": 13
},
{
"name": "blk.27.attn_q.bias",
"offset": 38887874560,
"shape": [
3584
],
"size": 8830976,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 38887888896,
"shape": [
3584,
3584
],
"size": 14336,
"type": 13
},
{
"name": "blk.27.attn_v.bias",
"offset": 38896719872,
"shape": [
512
],
"size": 8830976,
"type": 0
},
{
"name": "blk.27.attn_v.weight",
"offset": 38896721920,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "output.weight",
"offset": 38898227200,
"shape": [
3584,
151936
],
"size": 1505280,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 39344919040,
"shape": [
3584
],
"size": 446691840,
"type": 0
},
{
"name": "blk.27.ffn_down_exps.weight",
"offset": 39344933376,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.27.ffn_gate_exps.weight",
"offset": 39826622976,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 13
},
{
"name": "blk.27.ffn_up_exps.weight",
"offset": 40230324736,
"shape": [
3584,
2560,
64
],
"size": 403701760,
"type": 13
},
{
"name": "blk.27.ffn_down_shexp.weight",
"offset": 40634026496,
"shape": [
20480,
3584
],
"size": 403701760,
"type": 14
},
{
"name": "blk.27.ffn_gate_shexp.weight",
"offset": 40694237696,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 13
},
{
"name": "blk.27.ffn_up_shexp.weight",
"offset": 40744700416,
"shape": [
3584,
20480
],
"size": 50462720,
"type": 13
},
{
"name": "blk.27.ffn_gate_inp_shexp.weight",
"offset": 40795163136,
"shape": [
3584
],
"size": 50462720,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 40795177472,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 40795191808,
"shape": [
3584
],
"size": 14336,
"type": 0
}
],
"version": 3
}