Qwen2 MOE 57B
297 Pulls Updated 3 weeks ago
96023db93073 · 35GB
{
"metadata": {
"general.architecture": "qwen2moe",
"general.file_type": 15,
"general.name": "Qwen2-57B-A14B-Instruct",
"general.quantization_version": 2,
"general.source.url": "https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct",
"general.url": "https://huggingface.co/mradermacher/Qwen2-57B-A14B-Instruct-GGUF",
"mradermacher.convert_type": "hf",
"mradermacher.quantize_version": "2",
"mradermacher.quantized_at": "2024-06-07T16:21:16+02:00",
"mradermacher.quantized_by": "mradermacher",
"mradermacher.quantized_on": "kaos",
"qwen2moe.attention.head_count": 28,
"qwen2moe.attention.head_count_kv": 4,
"qwen2moe.attention.layer_norm_rms_epsilon": 0.000001,
"qwen2moe.block_count": 28,
"qwen2moe.context_length": 32768,
"qwen2moe.embedding_length": 3584,
"qwen2moe.expert_count": 64,
"qwen2moe.expert_used_count": 8,
"qwen2moe.feed_forward_length": 20480,
"qwen2moe.rope.freq_base": 1000000,
"tokenizer.ggml.bos_token_id": 151643,
"tokenizer.ggml.eos_token_id": 151645,
"tokenizer.ggml.merges": "... (151387 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 151643,
"tokenizer.ggml.pre": "qwen2",
"tokenizer.ggml.token_type": "... (151936 values)",
"tokenizer.ggml.tokens": "... (151936 values)"
},
"num_params": 57408658944,
"tensors": [
{
"name": "blk.0.ffn_gate_inp.weight",
"offset": 306302976,
"shape": [
3584,
64
],
"size": 306302976,
"type": 0
},
{
"name": "blk.0.attn_k.bias",
"offset": 307220480,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 307222528,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.0.attn_output.weight",
"offset": 308254720,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.0.attn_q.bias",
"offset": 315480064,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.0.attn_q.weight",
"offset": 315494400,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.0.attn_v.bias",
"offset": 322719744,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.0.attn_v.weight",
"offset": 322721792,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.0.attn_norm.weight",
"offset": 324227072,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.0.ffn_down_exps.weight",
"offset": 324241408,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.0.ffn_gate_exps.weight",
"offset": 805931008,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.0.ffn_up_exps.weight",
"offset": 1136232448,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.0.ffn_down_shexp.weight",
"offset": 1466533888,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.0.ffn_gate_shexp.weight",
"offset": 1526745088,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.0.ffn_up_shexp.weight",
"offset": 1568032768,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.0.ffn_gate_inp_shexp.weight",
"offset": 1609320448,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1609334784,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.ffn_gate_inp.weight",
"offset": 1609349120,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.attn_k.bias",
"offset": 1610266624,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 1610268672,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.1.attn_output.weight",
"offset": 1611300864,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.1.attn_q.bias",
"offset": 1618526208,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 1618540544,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.1.attn_v.bias",
"offset": 1625765888,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.1.attn_v.weight",
"offset": 1625767936,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 1627273216,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.1.ffn_down_exps.weight",
"offset": 1627287552,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.1.ffn_gate_exps.weight",
"offset": 2108977152,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.1.ffn_up_exps.weight",
"offset": 2439278592,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.1.ffn_down_shexp.weight",
"offset": 2769580032,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.1.ffn_gate_shexp.weight",
"offset": 2829791232,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.1.ffn_up_shexp.weight",
"offset": 2871078912,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.1.ffn_gate_inp_shexp.weight",
"offset": 2912366592,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 2912380928,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.ffn_gate_inp.weight",
"offset": 2912395264,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.attn_k.bias",
"offset": 2913312768,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 2913314816,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.2.attn_output.weight",
"offset": 2914347008,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.2.attn_q.bias",
"offset": 2921572352,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 2921586688,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.2.attn_v.bias",
"offset": 2928812032,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.2.attn_v.weight",
"offset": 2928814080,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 2930319360,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.2.ffn_down_exps.weight",
"offset": 2930333696,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.2.ffn_gate_exps.weight",
"offset": 3412023296,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.2.ffn_up_exps.weight",
"offset": 3742324736,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.2.ffn_down_shexp.weight",
"offset": 4072626176,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.2.ffn_gate_shexp.weight",
"offset": 4132837376,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.2.ffn_up_shexp.weight",
"offset": 4174125056,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.2.ffn_gate_inp_shexp.weight",
"offset": 4215412736,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 4215427072,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.ffn_gate_inp.weight",
"offset": 4215441408,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.attn_k.bias",
"offset": 4216358912,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 4216360960,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.3.attn_output.weight",
"offset": 4217393152,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.3.attn_q.bias",
"offset": 4224618496,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 4224632832,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.3.attn_v.bias",
"offset": 4231858176,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.3.attn_v.weight",
"offset": 4231860224,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.3.attn_norm.weight",
"offset": 4232892416,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.3.ffn_down_exps.weight",
"offset": 4232906752,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.3.ffn_gate_exps.weight",
"offset": 4563208192,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.3.ffn_up_exps.weight",
"offset": 4893509632,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.3.ffn_down_shexp.weight",
"offset": 5223811072,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.3.ffn_gate_shexp.weight",
"offset": 5265098752,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.3.ffn_up_shexp.weight",
"offset": 5306386432,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.3.ffn_gate_inp_shexp.weight",
"offset": 5347674112,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 5347688448,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.ffn_gate_inp.weight",
"offset": 5347702784,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.attn_k.bias",
"offset": 5348620288,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 5348622336,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.4.attn_output.weight",
"offset": 5349654528,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.4.attn_q.bias",
"offset": 5356879872,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 5356894208,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.4.attn_v.bias",
"offset": 5364119552,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.4.attn_v.weight",
"offset": 5364121600,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.4.attn_norm.weight",
"offset": 5365153792,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.4.ffn_down_exps.weight",
"offset": 5365168128,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.4.ffn_gate_exps.weight",
"offset": 5695469568,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.4.ffn_up_exps.weight",
"offset": 6025771008,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.4.ffn_down_shexp.weight",
"offset": 6356072448,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.4.ffn_gate_shexp.weight",
"offset": 6397360128,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.4.ffn_up_shexp.weight",
"offset": 6438647808,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.4.ffn_gate_inp_shexp.weight",
"offset": 6479935488,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 6479949824,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.ffn_gate_inp.weight",
"offset": 6479964160,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.attn_k.bias",
"offset": 6480881664,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 6480883712,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.5.attn_output.weight",
"offset": 6481915904,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.5.attn_q.bias",
"offset": 6489141248,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 6489155584,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.5.attn_v.bias",
"offset": 6496380928,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.5.attn_v.weight",
"offset": 6496382976,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 6497888256,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.5.ffn_down_exps.weight",
"offset": 6497902592,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.5.ffn_gate_exps.weight",
"offset": 6979592192,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.5.ffn_up_exps.weight",
"offset": 7309893632,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.5.ffn_down_shexp.weight",
"offset": 7640195072,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.5.ffn_gate_shexp.weight",
"offset": 7700406272,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.5.ffn_up_shexp.weight",
"offset": 7741693952,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.5.ffn_gate_inp_shexp.weight",
"offset": 7782981632,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 7782995968,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.ffn_gate_inp.weight",
"offset": 7783010304,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.attn_k.bias",
"offset": 7783927808,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 7783929856,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.6.attn_output.weight",
"offset": 7784962048,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.6.attn_q.bias",
"offset": 7792187392,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 7792201728,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.6.attn_v.bias",
"offset": 7799427072,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.6.attn_v.weight",
"offset": 7799429120,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.6.attn_norm.weight",
"offset": 7800461312,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.6.ffn_down_exps.weight",
"offset": 7800475648,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.6.ffn_gate_exps.weight",
"offset": 8130777088,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.6.ffn_up_exps.weight",
"offset": 8461078528,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.6.ffn_down_shexp.weight",
"offset": 8791379968,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.6.ffn_gate_shexp.weight",
"offset": 8832667648,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.6.ffn_up_shexp.weight",
"offset": 8873955328,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.6.ffn_gate_inp_shexp.weight",
"offset": 8915243008,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 8915257344,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.ffn_gate_inp.weight",
"offset": 8915271680,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.attn_k.bias",
"offset": 8916189184,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 8916191232,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.7.attn_output.weight",
"offset": 8917223424,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.7.attn_q.bias",
"offset": 8924448768,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 8924463104,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.7.attn_v.bias",
"offset": 8931688448,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.7.attn_v.weight",
"offset": 8931690496,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.7.attn_norm.weight",
"offset": 8932722688,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.7.ffn_down_exps.weight",
"offset": 8932737024,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.7.ffn_gate_exps.weight",
"offset": 9263038464,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.7.ffn_up_exps.weight",
"offset": 9593339904,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.7.ffn_down_shexp.weight",
"offset": 9923641344,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.7.ffn_gate_shexp.weight",
"offset": 9964929024,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.7.ffn_up_shexp.weight",
"offset": 10006216704,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.7.ffn_gate_inp_shexp.weight",
"offset": 10047504384,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 10047518720,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.ffn_gate_inp.weight",
"offset": 10047533056,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.attn_k.bias",
"offset": 10048450560,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 10048452608,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.8.attn_output.weight",
"offset": 10049484800,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.8.attn_q.bias",
"offset": 10056710144,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 10056724480,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.8.attn_v.bias",
"offset": 10063949824,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.8.attn_v.weight",
"offset": 10063951872,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 10065457152,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.8.ffn_down_exps.weight",
"offset": 10065471488,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.8.ffn_gate_exps.weight",
"offset": 10547161088,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.8.ffn_up_exps.weight",
"offset": 10877462528,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.8.ffn_down_shexp.weight",
"offset": 11207763968,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.8.ffn_gate_shexp.weight",
"offset": 11267975168,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.8.ffn_up_shexp.weight",
"offset": 11309262848,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.8.ffn_gate_inp_shexp.weight",
"offset": 11350550528,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 11350564864,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.ffn_gate_inp.weight",
"offset": 11350579200,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.attn_k.bias",
"offset": 11351496704,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 11351498752,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.9.attn_output.weight",
"offset": 11352530944,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.9.attn_q.bias",
"offset": 11359756288,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 11359770624,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.9.attn_v.bias",
"offset": 11366995968,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.9.attn_v.weight",
"offset": 11366998016,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.10.ffn_gate_inp.weight",
"offset": 11368030208,
"shape": [
3584,
64
],
"size": 1032192,
"type": 0
},
{
"name": "blk.10.attn_k.bias",
"offset": 11368947712,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 11368949760,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.10.attn_output.weight",
"offset": 11369981952,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.10.attn_q.bias",
"offset": 11377207296,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 11377221632,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.10.attn_v.bias",
"offset": 11384446976,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.10.attn_v.weight",
"offset": 11384449024,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.9.attn_norm.weight",
"offset": 11385481216,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.9.ffn_down_exps.weight",
"offset": 11385495552,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.9.ffn_gate_exps.weight",
"offset": 11715796992,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.9.ffn_up_exps.weight",
"offset": 12046098432,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.9.ffn_down_shexp.weight",
"offset": 12376399872,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.9.ffn_gate_shexp.weight",
"offset": 12417687552,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.9.ffn_up_shexp.weight",
"offset": 12458975232,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.9.ffn_gate_inp_shexp.weight",
"offset": 12500262912,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 12500277248,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.attn_norm.weight",
"offset": 12500291584,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.ffn_down_exps.weight",
"offset": 12500305920,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.10.ffn_gate_exps.weight",
"offset": 12830607360,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.10.ffn_up_exps.weight",
"offset": 13160908800,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.10.ffn_down_shexp.weight",
"offset": 13491210240,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.10.ffn_gate_shexp.weight",
"offset": 13532497920,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.10.ffn_up_shexp.weight",
"offset": 13573785600,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.10.ffn_gate_inp_shexp.weight",
"offset": 13615073280,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 13615087616,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.ffn_gate_inp.weight",
"offset": 13615101952,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.attn_k.bias",
"offset": 13616019456,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 13616021504,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.11.attn_output.weight",
"offset": 13617053696,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.11.attn_q.bias",
"offset": 13624279040,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 13624293376,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.11.attn_v.bias",
"offset": 13631518720,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.11.attn_v.weight",
"offset": 13631520768,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 13633026048,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.11.ffn_down_exps.weight",
"offset": 13633040384,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.11.ffn_gate_exps.weight",
"offset": 14114729984,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.11.ffn_up_exps.weight",
"offset": 14445031424,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.11.ffn_down_shexp.weight",
"offset": 14775332864,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.11.ffn_gate_shexp.weight",
"offset": 14835544064,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.11.ffn_up_shexp.weight",
"offset": 14876831744,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.11.ffn_gate_inp_shexp.weight",
"offset": 14918119424,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 14918133760,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.ffn_gate_inp.weight",
"offset": 14918148096,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.attn_k.bias",
"offset": 14919065600,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 14919067648,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.12.attn_output.weight",
"offset": 14920099840,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.12.attn_q.bias",
"offset": 14927325184,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 14927339520,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.12.attn_v.bias",
"offset": 14934564864,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.12.attn_v.weight",
"offset": 14934566912,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.12.attn_norm.weight",
"offset": 14935599104,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.12.ffn_down_exps.weight",
"offset": 14935613440,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.12.ffn_gate_exps.weight",
"offset": 15265914880,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.12.ffn_up_exps.weight",
"offset": 15596216320,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.12.ffn_down_shexp.weight",
"offset": 15926517760,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.12.ffn_gate_shexp.weight",
"offset": 15967805440,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.12.ffn_up_shexp.weight",
"offset": 16009093120,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.12.ffn_gate_inp_shexp.weight",
"offset": 16050380800,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 16050395136,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.ffn_gate_inp.weight",
"offset": 16050409472,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.attn_k.bias",
"offset": 16051326976,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 16051329024,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.13.attn_output.weight",
"offset": 16052361216,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.13.attn_q.bias",
"offset": 16059586560,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 16059600896,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.13.attn_v.bias",
"offset": 16066826240,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.13.attn_v.weight",
"offset": 16066828288,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.13.attn_norm.weight",
"offset": 16067860480,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.13.ffn_down_exps.weight",
"offset": 16067874816,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.13.ffn_gate_exps.weight",
"offset": 16398176256,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.13.ffn_up_exps.weight",
"offset": 16728477696,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.13.ffn_down_shexp.weight",
"offset": 17058779136,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.13.ffn_gate_shexp.weight",
"offset": 17100066816,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.13.ffn_up_shexp.weight",
"offset": 17141354496,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.13.ffn_gate_inp_shexp.weight",
"offset": 17182642176,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 17182656512,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.ffn_gate_inp.weight",
"offset": 17182670848,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.attn_k.bias",
"offset": 17183588352,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 17183590400,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.14.attn_output.weight",
"offset": 17184622592,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.14.attn_q.bias",
"offset": 17191847936,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 17191862272,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.14.attn_v.bias",
"offset": 17199087616,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.14.attn_v.weight",
"offset": 17199089664,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 17200594944,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.14.ffn_down_exps.weight",
"offset": 17200609280,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.14.ffn_gate_exps.weight",
"offset": 17682298880,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.14.ffn_up_exps.weight",
"offset": 18012600320,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.14.ffn_down_shexp.weight",
"offset": 18342901760,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.14.ffn_gate_shexp.weight",
"offset": 18403112960,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.14.ffn_up_shexp.weight",
"offset": 18444400640,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.14.ffn_gate_inp_shexp.weight",
"offset": 18485688320,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 18485702656,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.ffn_gate_inp.weight",
"offset": 18485716992,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.attn_k.bias",
"offset": 18486634496,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 18486636544,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.15.attn_output.weight",
"offset": 18487668736,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.15.attn_q.bias",
"offset": 18494894080,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 18494908416,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.15.attn_v.bias",
"offset": 18502133760,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.15.attn_v.weight",
"offset": 18502135808,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.15.attn_norm.weight",
"offset": 18503168000,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.15.ffn_down_exps.weight",
"offset": 18503182336,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.15.ffn_gate_exps.weight",
"offset": 18833483776,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.15.ffn_up_exps.weight",
"offset": 19163785216,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.15.ffn_down_shexp.weight",
"offset": 19494086656,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.15.ffn_gate_shexp.weight",
"offset": 19535374336,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.15.ffn_up_shexp.weight",
"offset": 19576662016,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.15.ffn_gate_inp_shexp.weight",
"offset": 19617949696,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 19617964032,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.ffn_gate_inp.weight",
"offset": 19617978368,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.attn_k.bias",
"offset": 19618895872,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 19618897920,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.16.attn_output.weight",
"offset": 19619930112,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.16.attn_q.bias",
"offset": 19627155456,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 19627169792,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.16.attn_v.bias",
"offset": 19634395136,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.16.attn_v.weight",
"offset": 19634397184,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 19635429376,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.16.ffn_down_exps.weight",
"offset": 19635443712,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.16.ffn_gate_exps.weight",
"offset": 19965745152,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.16.ffn_up_exps.weight",
"offset": 20296046592,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.16.ffn_down_shexp.weight",
"offset": 20626348032,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.16.ffn_gate_shexp.weight",
"offset": 20667635712,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.16.ffn_up_shexp.weight",
"offset": 20708923392,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.16.ffn_gate_inp_shexp.weight",
"offset": 20750211072,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 20750225408,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.ffn_gate_inp.weight",
"offset": 20750239744,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.attn_k.bias",
"offset": 20751157248,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 20751159296,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.17.attn_output.weight",
"offset": 20752191488,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.17.attn_q.bias",
"offset": 20759416832,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 20759431168,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.17.attn_v.bias",
"offset": 20766656512,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.17.attn_v.weight",
"offset": 20766658560,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 20768163840,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.17.ffn_down_exps.weight",
"offset": 20768178176,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.17.ffn_gate_exps.weight",
"offset": 21249867776,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.17.ffn_up_exps.weight",
"offset": 21580169216,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.17.ffn_down_shexp.weight",
"offset": 21910470656,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.17.ffn_gate_shexp.weight",
"offset": 21970681856,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.17.ffn_up_shexp.weight",
"offset": 22011969536,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.17.ffn_gate_inp_shexp.weight",
"offset": 22053257216,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 22053271552,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.ffn_gate_inp.weight",
"offset": 22053285888,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.attn_k.bias",
"offset": 22054203392,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 22054205440,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.18.attn_output.weight",
"offset": 22055237632,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.18.attn_q.bias",
"offset": 22062462976,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 22062477312,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.18.attn_v.bias",
"offset": 22069702656,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.18.attn_v.weight",
"offset": 22069704704,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.18.attn_norm.weight",
"offset": 22070736896,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.18.ffn_down_exps.weight",
"offset": 22070751232,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.18.ffn_gate_exps.weight",
"offset": 22401052672,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.18.ffn_up_exps.weight",
"offset": 22731354112,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.18.ffn_down_shexp.weight",
"offset": 23061655552,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.18.ffn_gate_shexp.weight",
"offset": 23102943232,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.18.ffn_up_shexp.weight",
"offset": 23144230912,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.18.ffn_gate_inp_shexp.weight",
"offset": 23185518592,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 23185532928,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.ffn_gate_inp.weight",
"offset": 23185547264,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.attn_k.bias",
"offset": 23186464768,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 23186466816,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.19.attn_output.weight",
"offset": 23187499008,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.19.attn_q.bias",
"offset": 23194724352,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 23194738688,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.19.attn_v.bias",
"offset": 23201964032,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.19.attn_v.weight",
"offset": 23201966080,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 23202998272,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.19.ffn_down_exps.weight",
"offset": 23203012608,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.19.ffn_gate_exps.weight",
"offset": 23533314048,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.19.ffn_up_exps.weight",
"offset": 23863615488,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.19.ffn_down_shexp.weight",
"offset": 24193916928,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.19.ffn_gate_shexp.weight",
"offset": 24235204608,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.19.ffn_up_shexp.weight",
"offset": 24276492288,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.19.ffn_gate_inp_shexp.weight",
"offset": 24317779968,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 24317794304,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.ffn_gate_inp.weight",
"offset": 24317808640,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.attn_k.bias",
"offset": 24318726144,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 24318728192,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.20.attn_output.weight",
"offset": 24319760384,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.20.attn_q.bias",
"offset": 24326985728,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 24327000064,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.20.attn_v.bias",
"offset": 24334225408,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.20.attn_v.weight",
"offset": 24334227456,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 24335732736,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.20.ffn_down_exps.weight",
"offset": 24335747072,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.20.ffn_gate_exps.weight",
"offset": 24817436672,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.20.ffn_up_exps.weight",
"offset": 25147738112,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.20.ffn_down_shexp.weight",
"offset": 25478039552,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.20.ffn_gate_shexp.weight",
"offset": 25538250752,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.20.ffn_up_shexp.weight",
"offset": 25579538432,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.20.ffn_gate_inp_shexp.weight",
"offset": 25620826112,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 25620840448,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.21.ffn_gate_inp.weight",
"offset": 25620854784,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.21.attn_k.bias",
"offset": 25621772288,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 25621774336,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.21.attn_output.weight",
"offset": 25622806528,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.21.attn_q.bias",
"offset": 25630031872,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 25630046208,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.21.attn_v.bias",
"offset": 25637271552,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.21.attn_v.weight",
"offset": 25637273600,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.21.attn_norm.weight",
"offset": 25638305792,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.21.ffn_down_exps.weight",
"offset": 25638320128,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.21.ffn_gate_exps.weight",
"offset": 25968621568,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.21.ffn_up_exps.weight",
"offset": 26298923008,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.21.ffn_down_shexp.weight",
"offset": 26629224448,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.21.ffn_gate_shexp.weight",
"offset": 26670512128,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.21.ffn_up_shexp.weight",
"offset": 26711799808,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.21.ffn_gate_inp_shexp.weight",
"offset": 26753087488,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 26753101824,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.ffn_gate_inp.weight",
"offset": 26753116160,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.attn_k.bias",
"offset": 26754033664,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 26754035712,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.22.attn_output.weight",
"offset": 26755067904,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.22.attn_q.bias",
"offset": 26762293248,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 26762307584,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.22.attn_v.bias",
"offset": 26769532928,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.22.attn_v.weight",
"offset": 26769534976,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 26770567168,
"shape": [
3584
],
"size": 1032192,
"type": 0
},
{
"name": "blk.22.ffn_down_exps.weight",
"offset": 26770581504,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 12
},
{
"name": "blk.22.ffn_gate_exps.weight",
"offset": 27100882944,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.22.ffn_up_exps.weight",
"offset": 27431184384,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.22.ffn_down_shexp.weight",
"offset": 27761485824,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 12
},
{
"name": "blk.22.ffn_gate_shexp.weight",
"offset": 27802773504,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.22.ffn_up_shexp.weight",
"offset": 27844061184,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.22.ffn_gate_inp_shexp.weight",
"offset": 27885348864,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 27885363200,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.ffn_gate_inp.weight",
"offset": 27885377536,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.attn_k.bias",
"offset": 27886295040,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 27886297088,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.23.attn_output.weight",
"offset": 27887329280,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.23.attn_q.bias",
"offset": 27894554624,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 27894568960,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.23.attn_v.bias",
"offset": 27901794304,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.23.attn_v.weight",
"offset": 27901796352,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 27903301632,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.23.ffn_down_exps.weight",
"offset": 27903315968,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.23.ffn_gate_exps.weight",
"offset": 28385005568,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.23.ffn_up_exps.weight",
"offset": 28715307008,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.23.ffn_down_shexp.weight",
"offset": 29045608448,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.23.ffn_gate_shexp.weight",
"offset": 29105819648,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.23.ffn_up_shexp.weight",
"offset": 29147107328,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.23.ffn_gate_inp_shexp.weight",
"offset": 29188395008,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 29188409344,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.ffn_gate_inp.weight",
"offset": 29188423680,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.attn_k.bias",
"offset": 29189341184,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 29189343232,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.24.attn_output.weight",
"offset": 29190375424,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.24.attn_q.bias",
"offset": 29197600768,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 29197615104,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.24.attn_v.bias",
"offset": 29204840448,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.24.attn_v.weight",
"offset": 29204842496,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 29206347776,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.24.ffn_down_exps.weight",
"offset": 29206362112,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.24.ffn_gate_exps.weight",
"offset": 29688051712,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.24.ffn_up_exps.weight",
"offset": 30018353152,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.24.ffn_down_shexp.weight",
"offset": 30348654592,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.24.ffn_gate_shexp.weight",
"offset": 30408865792,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.24.ffn_up_shexp.weight",
"offset": 30450153472,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.24.ffn_gate_inp_shexp.weight",
"offset": 30491441152,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 30491455488,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.ffn_gate_inp.weight",
"offset": 30491469824,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.attn_k.bias",
"offset": 30492387328,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 30492389376,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.25.attn_output.weight",
"offset": 30493421568,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.25.attn_q.bias",
"offset": 30500646912,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 30500661248,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.25.attn_v.bias",
"offset": 30507886592,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.25.attn_v.weight",
"offset": 30507888640,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 30509393920,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.25.ffn_down_exps.weight",
"offset": 30509408256,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.25.ffn_gate_exps.weight",
"offset": 30991097856,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.25.ffn_up_exps.weight",
"offset": 31321399296,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.25.ffn_down_shexp.weight",
"offset": 31651700736,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.25.ffn_gate_shexp.weight",
"offset": 31711911936,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.25.ffn_up_shexp.weight",
"offset": 31753199616,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.25.ffn_gate_inp_shexp.weight",
"offset": 31794487296,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 31794501632,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.ffn_gate_inp.weight",
"offset": 31794515968,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.attn_k.bias",
"offset": 31795433472,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 31795435520,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.26.attn_output.weight",
"offset": 31796467712,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.26.attn_q.bias",
"offset": 31803693056,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 31803707392,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.26.attn_v.bias",
"offset": 31810932736,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.26.attn_v.weight",
"offset": 31810934784,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 31812440064,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.26.ffn_down_exps.weight",
"offset": 31812454400,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.26.ffn_gate_exps.weight",
"offset": 32294144000,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.26.ffn_up_exps.weight",
"offset": 32624445440,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.26.ffn_down_shexp.weight",
"offset": 32954746880,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.26.ffn_gate_shexp.weight",
"offset": 33014958080,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.26.ffn_up_shexp.weight",
"offset": 33056245760,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.26.ffn_gate_inp_shexp.weight",
"offset": 33097533440,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 33097547776,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.ffn_gate_inp.weight",
"offset": 33097562112,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.attn_k.bias",
"offset": 33098479616,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 33098481664,
"shape": [
3584,
512
],
"size": 2048,
"type": 12
},
{
"name": "blk.27.attn_output.weight",
"offset": 33099513856,
"shape": [
3584,
3584
],
"size": 1032192,
"type": 12
},
{
"name": "blk.27.attn_q.bias",
"offset": 33106739200,
"shape": [
3584
],
"size": 7225344,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 33106753536,
"shape": [
3584,
3584
],
"size": 14336,
"type": 12
},
{
"name": "blk.27.attn_v.bias",
"offset": 33113978880,
"shape": [
512
],
"size": 7225344,
"type": 0
},
{
"name": "blk.27.attn_v.weight",
"offset": 33113980928,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "output.weight",
"offset": 33115486208,
"shape": [
3584,
151936
],
"size": 1505280,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 33562178048,
"shape": [
3584
],
"size": 446691840,
"type": 0
},
{
"name": "blk.27.ffn_down_exps.weight",
"offset": 33562192384,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.27.ffn_gate_exps.weight",
"offset": 34043881984,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 12
},
{
"name": "blk.27.ffn_up_exps.weight",
"offset": 34374183424,
"shape": [
3584,
2560,
64
],
"size": 330301440,
"type": 12
},
{
"name": "blk.27.ffn_down_shexp.weight",
"offset": 34704484864,
"shape": [
20480,
3584
],
"size": 330301440,
"type": 14
},
{
"name": "blk.27.ffn_gate_shexp.weight",
"offset": 34764696064,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 12
},
{
"name": "blk.27.ffn_up_shexp.weight",
"offset": 34805983744,
"shape": [
3584,
20480
],
"size": 41287680,
"type": 12
},
{
"name": "blk.27.ffn_gate_inp_shexp.weight",
"offset": 34847271424,
"shape": [
3584
],
"size": 41287680,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 34847285760,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 34847300096,
"shape": [
3584
],
"size": 14336,
"type": 0
}
],
"version": 3
}