Qwen2 MOE 57B
298 Pulls Updated 3 weeks ago
59a9e0577b00 · 47GB
{
"metadata": {
"general.architecture": "qwen2moe",
"general.file_type": 18,
"general.name": "Qwen2-57B-A14B-Instruct",
"general.quantization_version": 2,
"general.source.url": "https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct",
"general.url": "https://huggingface.co/mradermacher/Qwen2-57B-A14B-Instruct-GGUF",
"mradermacher.convert_type": "hf",
"mradermacher.quantize_version": "2",
"mradermacher.quantized_at": "2024-06-07T15:15:52+02:00",
"mradermacher.quantized_by": "mradermacher",
"mradermacher.quantized_on": "kaos",
"qwen2moe.attention.head_count": 28,
"qwen2moe.attention.head_count_kv": 4,
"qwen2moe.attention.layer_norm_rms_epsilon": 0.000001,
"qwen2moe.block_count": 28,
"qwen2moe.context_length": 32768,
"qwen2moe.embedding_length": 3584,
"qwen2moe.expert_count": 64,
"qwen2moe.expert_used_count": 8,
"qwen2moe.feed_forward_length": 20480,
"qwen2moe.rope.freq_base": 1000000,
"tokenizer.ggml.bos_token_id": 151643,
"tokenizer.ggml.eos_token_id": 151645,
"tokenizer.ggml.merges": "... (151387 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 151643,
"tokenizer.ggml.pre": "qwen2",
"tokenizer.ggml.token_type": "... (151936 values)",
"tokenizer.ggml.tokens": "... (151936 values)"
},
"num_params": 57408658944,
"tensors": [
{
"name": "blk.0.ffn_gate_inp.weight",
"offset": 446691840,
"shape": [
3584,
64
],
"size": 446691840,
"type": 0
},
{
"name": "blk.0.attn_k.bias",
"offset": 447609344,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 447611392,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 449116672,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.0.attn_q.bias",
"offset": 459653632,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.0.attn_q.weight",
"offset": 459667968,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.0.attn_v.bias",
"offset": 470204928,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.0.attn_v.weight",
"offset": 470206976,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.0.attn_norm.weight",
"offset": 471712256,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.0.ffn_down_exps.weight",
"offset": 471726592,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.0.ffn_gate_exps.weight",
"offset": 953416192,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.0.ffn_up_exps.weight",
"offset": 1435105792,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.0.ffn_down_shexp.weight",
"offset": 1916795392,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.0.ffn_gate_shexp.weight",
"offset": 1977006592,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.0.ffn_up_shexp.weight",
"offset": 2037217792,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.0.ffn_gate_inp_shexp.weight",
"offset": 2097428992,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 2097443328,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.ffn_gate_inp.weight",
"offset": 2097457664,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.attn_k.bias",
"offset": 2098375168,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 2098377216,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 2099882496,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.1.attn_q.bias",
"offset": 2110419456,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 2110433792,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.1.attn_v.bias",
"offset": 2120970752,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.1.attn_v.weight",
"offset": 2120972800,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2122478080,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.1.ffn_down_exps.weight",
"offset": 2122492416,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.1.ffn_gate_exps.weight",
"offset": 2604182016,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.1.ffn_up_exps.weight",
"offset": 3085871616,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.1.ffn_down_shexp.weight",
"offset": 3567561216,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.1.ffn_gate_shexp.weight",
"offset": 3627772416,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.1.ffn_up_shexp.weight",
"offset": 3687983616,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.1.ffn_gate_inp_shexp.weight",
"offset": 3748194816,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 3748209152,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.ffn_gate_inp.weight",
"offset": 3748223488,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.attn_k.bias",
"offset": 3749140992,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 3749143040,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 3750648320,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.2.attn_q.bias",
"offset": 3761185280,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 3761199616,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.2.attn_v.bias",
"offset": 3771736576,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.2.attn_v.weight",
"offset": 3771738624,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 3773243904,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.2.ffn_down_exps.weight",
"offset": 3773258240,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.2.ffn_gate_exps.weight",
"offset": 4254947840,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.2.ffn_up_exps.weight",
"offset": 4736637440,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.2.ffn_down_shexp.weight",
"offset": 5218327040,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.2.ffn_gate_shexp.weight",
"offset": 5278538240,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.2.ffn_up_shexp.weight",
"offset": 5338749440,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.2.ffn_gate_inp_shexp.weight",
"offset": 5398960640,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 5398974976,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.ffn_gate_inp.weight",
"offset": 5398989312,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.attn_k.bias",
"offset": 5399906816,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 5399908864,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 5401414144,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.3.attn_q.bias",
"offset": 5411951104,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 5411965440,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.3.attn_v.bias",
"offset": 5422502400,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.3.attn_v.weight",
"offset": 5422504448,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 5424009728,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.3.ffn_down_exps.weight",
"offset": 5424024064,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.3.ffn_gate_exps.weight",
"offset": 5905713664,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.3.ffn_up_exps.weight",
"offset": 6387403264,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.3.ffn_down_shexp.weight",
"offset": 6869092864,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.3.ffn_gate_shexp.weight",
"offset": 6929304064,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.3.ffn_up_shexp.weight",
"offset": 6989515264,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.3.ffn_gate_inp_shexp.weight",
"offset": 7049726464,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 7049740800,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.ffn_gate_inp.weight",
"offset": 7049755136,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.attn_k.bias",
"offset": 7050672640,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 7050674688,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 7052179968,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.4.attn_q.bias",
"offset": 7062716928,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 7062731264,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.4.attn_v.bias",
"offset": 7073268224,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.4.attn_v.weight",
"offset": 7073270272,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 7074775552,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.4.ffn_down_exps.weight",
"offset": 7074789888,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.4.ffn_gate_exps.weight",
"offset": 7556479488,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.4.ffn_up_exps.weight",
"offset": 8038169088,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.4.ffn_down_shexp.weight",
"offset": 8519858688,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.4.ffn_gate_shexp.weight",
"offset": 8580069888,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.4.ffn_up_shexp.weight",
"offset": 8640281088,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.4.ffn_gate_inp_shexp.weight",
"offset": 8700492288,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 8700506624,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.ffn_gate_inp.weight",
"offset": 8700520960,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.attn_k.bias",
"offset": 8701438464,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 8701440512,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 8702945792,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.5.attn_q.bias",
"offset": 8713482752,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 8713497088,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.5.attn_v.bias",
"offset": 8724034048,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.5.attn_v.weight",
"offset": 8724036096,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 8725541376,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.5.ffn_down_exps.weight",
"offset": 8725555712,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.5.ffn_gate_exps.weight",
"offset": 9207245312,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.5.ffn_up_exps.weight",
"offset": 9688934912,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.5.ffn_down_shexp.weight",
"offset": 10170624512,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.5.ffn_gate_shexp.weight",
"offset": 10230835712,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.5.ffn_up_shexp.weight",
"offset": 10291046912,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.5.ffn_gate_inp_shexp.weight",
"offset": 10351258112,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 10351272448,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.ffn_gate_inp.weight",
"offset": 10351286784,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.attn_k.bias",
"offset": 10352204288,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 10352206336,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 10353711616,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.6.attn_q.bias",
"offset": 10364248576,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 10364262912,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.6.attn_v.bias",
"offset": 10374799872,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.6.attn_v.weight",
"offset": 10374801920,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.6.attn_norm.weight",
"offset": 10376307200,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.6.ffn_down_exps.weight",
"offset": 10376321536,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.6.ffn_gate_exps.weight",
"offset": 10858011136,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.6.ffn_up_exps.weight",
"offset": 11339700736,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.6.ffn_down_shexp.weight",
"offset": 11821390336,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.6.ffn_gate_shexp.weight",
"offset": 11881601536,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.6.ffn_up_shexp.weight",
"offset": 11941812736,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.6.ffn_gate_inp_shexp.weight",
"offset": 12002023936,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 12002038272,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.ffn_gate_inp.weight",
"offset": 12002052608,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.attn_k.bias",
"offset": 12002970112,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 12002972160,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 12004477440,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.7.attn_q.bias",
"offset": 12015014400,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 12015028736,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.7.attn_v.bias",
"offset": 12025565696,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.7.attn_v.weight",
"offset": 12025567744,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 12027073024,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.7.ffn_down_exps.weight",
"offset": 12027087360,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.7.ffn_gate_exps.weight",
"offset": 12508776960,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.7.ffn_up_exps.weight",
"offset": 12990466560,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.7.ffn_down_shexp.weight",
"offset": 13472156160,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.7.ffn_gate_shexp.weight",
"offset": 13532367360,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.7.ffn_up_shexp.weight",
"offset": 13592578560,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.7.ffn_gate_inp_shexp.weight",
"offset": 13652789760,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 13652804096,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.ffn_gate_inp.weight",
"offset": 13652818432,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.attn_k.bias",
"offset": 13653735936,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 13653737984,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 13655243264,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.8.attn_q.bias",
"offset": 13665780224,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 13665794560,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.8.attn_v.bias",
"offset": 13676331520,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.8.attn_v.weight",
"offset": 13676333568,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 13677838848,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.8.ffn_down_exps.weight",
"offset": 13677853184,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.8.ffn_gate_exps.weight",
"offset": 14159542784,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.8.ffn_up_exps.weight",
"offset": 14641232384,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.8.ffn_down_shexp.weight",
"offset": 15122921984,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.8.ffn_gate_shexp.weight",
"offset": 15183133184,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.8.ffn_up_shexp.weight",
"offset": 15243344384,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.8.ffn_gate_inp_shexp.weight",
"offset": 15303555584,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 15303569920,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.ffn_gate_inp.weight",
"offset": 15303584256,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.attn_k.bias",
"offset": 15304501760,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 15304503808,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 15306009088,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.9.attn_q.bias",
"offset": 15316546048,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 15316560384,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.9.attn_v.bias",
"offset": 15327097344,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.9.attn_v.weight",
"offset": 15327099392,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.10.ffn_gate_inp.weight",
"offset": 15328604672,
"shape": [
3584,
64
],
"size": 1505280,
"type": 0
},
{
"name": "blk.10.attn_k.bias",
"offset": 15329522176,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 15329524224,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.10.attn_output.weight",
"offset": 15331029504,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.10.attn_q.bias",
"offset": 15341566464,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 15341580800,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.10.attn_v.bias",
"offset": 15352117760,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.10.attn_v.weight",
"offset": 15352119808,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 15353625088,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.9.ffn_down_exps.weight",
"offset": 15353639424,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.9.ffn_gate_exps.weight",
"offset": 15835329024,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.9.ffn_up_exps.weight",
"offset": 16317018624,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.9.ffn_down_shexp.weight",
"offset": 16798708224,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.9.ffn_gate_shexp.weight",
"offset": 16858919424,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.9.ffn_up_shexp.weight",
"offset": 16919130624,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.9.ffn_gate_inp_shexp.weight",
"offset": 16979341824,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 16979356160,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.attn_norm.weight",
"offset": 16979370496,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.ffn_down_exps.weight",
"offset": 16979384832,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.10.ffn_gate_exps.weight",
"offset": 17461074432,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.10.ffn_up_exps.weight",
"offset": 17942764032,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.10.ffn_down_shexp.weight",
"offset": 18424453632,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.10.ffn_gate_shexp.weight",
"offset": 18484664832,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.10.ffn_up_shexp.weight",
"offset": 18544876032,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.10.ffn_gate_inp_shexp.weight",
"offset": 18605087232,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 18605101568,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.ffn_gate_inp.weight",
"offset": 18605115904,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.attn_k.bias",
"offset": 18606033408,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 18606035456,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.11.attn_output.weight",
"offset": 18607540736,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.11.attn_q.bias",
"offset": 18618077696,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 18618092032,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.11.attn_v.bias",
"offset": 18628628992,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.11.attn_v.weight",
"offset": 18628631040,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 18630136320,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.11.ffn_down_exps.weight",
"offset": 18630150656,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.11.ffn_gate_exps.weight",
"offset": 19111840256,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.11.ffn_up_exps.weight",
"offset": 19593529856,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.11.ffn_down_shexp.weight",
"offset": 20075219456,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.11.ffn_gate_shexp.weight",
"offset": 20135430656,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.11.ffn_up_shexp.weight",
"offset": 20195641856,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.11.ffn_gate_inp_shexp.weight",
"offset": 20255853056,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 20255867392,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.ffn_gate_inp.weight",
"offset": 20255881728,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.attn_k.bias",
"offset": 20256799232,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 20256801280,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 20258306560,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.12.attn_q.bias",
"offset": 20268843520,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 20268857856,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.12.attn_v.bias",
"offset": 20279394816,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.12.attn_v.weight",
"offset": 20279396864,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 20280902144,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.12.ffn_down_exps.weight",
"offset": 20280916480,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.12.ffn_gate_exps.weight",
"offset": 20762606080,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.12.ffn_up_exps.weight",
"offset": 21244295680,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.12.ffn_down_shexp.weight",
"offset": 21725985280,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.12.ffn_gate_shexp.weight",
"offset": 21786196480,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.12.ffn_up_shexp.weight",
"offset": 21846407680,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.12.ffn_gate_inp_shexp.weight",
"offset": 21906618880,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 21906633216,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.ffn_gate_inp.weight",
"offset": 21906647552,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.attn_k.bias",
"offset": 21907565056,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 21907567104,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.13.attn_output.weight",
"offset": 21909072384,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.13.attn_q.bias",
"offset": 21919609344,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "_gate_inp.weight\u0002\u0000\u0000\u0000",
"offset": 21919623680,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.13.attn_v.bias",
"offset": 21930160640,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.13.attn_v.weight",
"offset": 21930162688,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 21931667968,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.13.ffn_down_exps.weight",
"offset": 21931682304,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.13.ffn_gate_exps.weight",
"offset": 22413371904,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.13.ffn_up_exps.weight",
"offset": 22895061504,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.13.ffn_down_shexp.weight",
"offset": 23376751104,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.13.ffn_gate_shexp.weight",
"offset": 23436962304,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.13.ffn_up_shexp.weight",
"offset": 23497173504,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.13.ffn_gate_inp_shexp.weight",
"offset": 23557384704,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 23557399040,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.ffn_gate_inp.weight",
"offset": 23557413376,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.attn_k.bias",
"offset": 23558330880,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 23558332928,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.14.attn_output.weight",
"offset": 23559838208,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.14.attn_q.bias",
"offset": 23570375168,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 23570389504,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.14.attn_v.bias",
"offset": 23580926464,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.14.attn_v.weight",
"offset": 23580928512,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 23582433792,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.14.ffn_down_exps.weight",
"offset": 23582448128,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.14.ffn_gate_exps.weight",
"offset": 24064137728,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.14.ffn_up_exps.weight",
"offset": 24545827328,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.14.ffn_down_shexp.weight",
"offset": 25027516928,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.14.ffn_gate_shexp.weight",
"offset": 25087728128,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.14.ffn_up_shexp.weight",
"offset": 25147939328,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.14.ffn_gate_inp_shexp.weight",
"offset": 25208150528,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 25208164864,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.ffn_gate_inp.weight",
"offset": 25208179200,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.attn_k.bias",
"offset": 25209096704,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 25209098752,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 25210604032,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.15.attn_q.bias",
"offset": 25221140992,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 25221155328,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.15.attn_v.bias",
"offset": 25231692288,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.15.attn_v.weight",
"offset": 25231694336,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 25233199616,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.15.ffn_down_exps.weight",
"offset": 25233213952,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.15.ffn_gate_exps.weight",
"offset": 25714903552,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.15.ffn_up_exps.weight",
"offset": 26196593152,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.15.ffn_down_shexp.weight",
"offset": 26678282752,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.15.ffn_gate_shexp.weight",
"offset": 26738493952,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.15.ffn_up_shexp.weight",
"offset": 26798705152,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.15.ffn_gate_inp_shexp.weight",
"offset": 26858916352,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 26858930688,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.ffn_gate_inp.weight",
"offset": 26858945024,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.attn_k.bias",
"offset": 26859862528,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 26859864576,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.16.attn_output.weight",
"offset": 26861369856,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.16.attn_q.bias",
"offset": 26871906816,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 26871921152,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.16.attn_v.bias",
"offset": 26882458112,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.16.attn_v.weight",
"offset": 26882460160,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 26883965440,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.16.ffn_down_exps.weight",
"offset": 26883979776,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.16.ffn_gate_exps.weight",
"offset": 27365669376,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.16.ffn_up_exps.weight",
"offset": 27847358976,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.16.ffn_down_shexp.weight",
"offset": 28329048576,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.16.ffn_gate_shexp.weight",
"offset": 28389259776,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.16.ffn_up_shexp.weight",
"offset": 28449470976,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.16.ffn_gate_inp_shexp.weight",
"offset": 28509682176,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 28509696512,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.ffn_gate_inp.weight",
"offset": 28509710848,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.attn_k.bias",
"offset": 28510628352,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 28510630400,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.17.attn_output.weight",
"offset": 28512135680,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.17.attn_q.bias",
"offset": 28522672640,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 28522686976,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.17.attn_v.bias",
"offset": 28533223936,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.17.attn_v.weight",
"offset": 28533225984,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 28534731264,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.17.ffn_down_exps.weight",
"offset": 28534745600,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.17.ffn_gate_exps.weight",
"offset": 29016435200,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.17.ffn_up_exps.weight",
"offset": 29498124800,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.17.ffn_down_shexp.weight",
"offset": 29979814400,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.17.ffn_gate_shexp.weight",
"offset": 30040025600,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.17.ffn_up_shexp.weight",
"offset": 30100236800,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.17.ffn_gate_inp_shexp.weight",
"offset": 30160448000,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 30160462336,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.ffn_gate_inp.weight",
"offset": 30160476672,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.attn_k.bias",
"offset": 30161394176,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 30161396224,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 30162901504,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.18.attn_q.bias",
"offset": 30173438464,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 30173452800,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.18.attn_v.bias",
"offset": 30183989760,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.18.attn_v.weight",
"offset": 30183991808,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 30185497088,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.18.ffn_down_exps.weight",
"offset": 30185511424,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.18.ffn_gate_exps.weight",
"offset": 30667201024,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.18.ffn_up_exps.weight",
"offset": 31148890624,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.18.ffn_down_shexp.weight",
"offset": 31630580224,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.18.ffn_gate_shexp.weight",
"offset": 31690791424,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.18.ffn_up_shexp.weight",
"offset": 31751002624,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.18.ffn_gate_inp_shexp.weight",
"offset": 31811213824,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 31811228160,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.ffn_gate_inp.weight",
"offset": 31811242496,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.attn_k.bias",
"offset": 31812160000,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 31812162048,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.19.attn_output.weight",
"offset": 31813667328,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.19.attn_q.bias",
"offset": 31824204288,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 31824218624,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.19.attn_v.bias",
"offset": 31834755584,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.19.attn_v.weight",
"offset": 31834757632,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.19.attn_norm.weight",
"offset": 31836262912,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.19.ffn_down_exps.weight",
"offset": 31836277248,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.19.ffn_gate_exps.weight",
"offset": 32317966848,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.19.ffn_up_exps.weight",
"offset": 32799656448,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.19.ffn_down_shexp.weight",
"offset": 33281346048,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.19.ffn_gate_shexp.weight",
"offset": 33341557248,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.19.ffn_up_shexp.weight",
"offset": 33401768448,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.19.ffn_gate_inp_shexp.weight",
"offset": 33461979648,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 33461993984,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.ffn_gate_inp.weight",
"offset": 33462008320,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.attn_k.bias",
"offset": 33462925824,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 33462927872,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.20.attn_output.weight",
"offset": 33464433152,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.20.attn_q.bias",
"offset": 33474970112,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 33474984448,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.20.attn_v.bias",
"offset": 33485521408,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.20.attn_v.weight",
"offset": 33485523456,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 33487028736,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.20.ffn_down_exps.weight",
"offset": 33487043072,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.20.ffn_gate_exps.weight",
"offset": 33968732672,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.20.ffn_up_exps.weight",
"offset": 34450422272,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.20.ffn_down_shexp.weight",
"offset": 34932111872,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.20.ffn_gate_shexp.weight",
"offset": 34992323072,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.20.ffn_up_shexp.weight",
"offset": 35052534272,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.20.ffn_gate_inp_shexp.weight",
"offset": 35112745472,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 35112759808,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "CE\u0019\ufffdMXeM\ufffd\u0001\ufffd\u0019\ufffd\u001f\ufffd\u0003\ufffdxw\ufffd\ufffdue\ufffd\ufffd\u0014",
"offset": 35112774144,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.21.attn_k.bias",
"offset": 35113691648,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 35113693696,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 35115198976,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.21.attn_q.bias",
"offset": 35125735936,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 35125750272,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.21.attn_v.bias",
"offset": 35136287232,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.21.attn_v.weight",
"offset": 35136289280,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 35137794560,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.21.ffn_down_exps.weight",
"offset": 35137808896,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.21.ffn_gate_exps.weight",
"offset": 35619498496,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.21.ffn_up_exps.weight",
"offset": 36101188096,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.21.ffn_down_shexp.weight",
"offset": 36582877696,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.21.ffn_gate_shexp.weight",
"offset": 36643088896,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.21.ffn_up_shexp.weight",
"offset": 36703300096,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.21.ffn_gate_inp_shexp.weight",
"offset": 36763511296,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 36763525632,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.ffn_gate_inp.weight",
"offset": 36763539968,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.attn_k.bias",
"offset": 36764457472,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 36764459520,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.22.attn_output.weight",
"offset": 36765964800,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.22.attn_q.bias",
"offset": 36776501760,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 36776516096,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.22.attn_v.bias",
"offset": 36787053056,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.22.attn_v.weight",
"offset": 36787055104,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.22.attn_norm.weight",
"offset": 36788560384,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.22.ffn_down_exps.weight",
"offset": 36788574720,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.22.ffn_gate_exps.weight",
"offset": 37270264320,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.22.ffn_up_exps.weight",
"offset": 37751953920,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.22.ffn_down_shexp.weight",
"offset": 38233643520,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.22.ffn_gate_shexp.weight",
"offset": 38293854720,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.22.ffn_up_shexp.weight",
"offset": 38354065920,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.22.ffn_gate_inp_shexp.weight",
"offset": 38414277120,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 38414291456,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.ffn_gate_inp.weight",
"offset": 38414305792,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.attn_k.bias",
"offset": 38415223296,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 38415225344,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.23.attn_output.weight",
"offset": 38416730624,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.23.attn_q.bias",
"offset": 38427267584,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 38427281920,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.23.attn_v.bias",
"offset": 38437818880,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.23.attn_v.weight",
"offset": 38437820928,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 38439326208,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.23.ffn_down_exps.weight",
"offset": 38439340544,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.23.ffn_gate_exps.weight",
"offset": 38921030144,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.23.ffn_up_exps.weight",
"offset": 39402719744,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.23.ffn_down_shexp.weight",
"offset": 39884409344,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.23.ffn_gate_shexp.weight",
"offset": 39944620544,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.23.ffn_up_shexp.weight",
"offset": 40004831744,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.23.ffn_gate_inp_shexp.weight",
"offset": 40065042944,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 40065057280,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.ffn_gate_inp.weight",
"offset": 40065071616,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.attn_k.bias",
"offset": 40065989120,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 40065991168,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 40067496448,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.24.attn_q.bias",
"offset": 40078033408,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 40078047744,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.24.attn_v.bias",
"offset": 40088584704,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.24.attn_v.weight",
"offset": 40088586752,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 40090092032,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.24.ffn_down_exps.weight",
"offset": 40090106368,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.24.ffn_gate_exps.weight",
"offset": 40571795968,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.24.ffn_up_exps.weight",
"offset": 41053485568,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.24.ffn_down_shexp.weight",
"offset": 41535175168,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.24.ffn_gate_shexp.weight",
"offset": 41595386368,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.24.ffn_up_shexp.weight",
"offset": 41655597568,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.24.ffn_gate_inp_shexp.weight",
"offset": 41715808768,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 41715823104,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.ffn_gate_inp.weight",
"offset": 41715837440,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.attn_k.bias",
"offset": 41716754944,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 41716756992,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.25.attn_output.weight",
"offset": 41718262272,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.25.attn_q.bias",
"offset": 41728799232,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 41728813568,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.25.attn_v.bias",
"offset": 41739350528,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.25.attn_v.weight",
"offset": 41739352576,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.25.attn_norm.weight",
"offset": 41740857856,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.25.ffn_down_exps.weight",
"offset": 41740872192,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.25.ffn_gate_exps.weight",
"offset": 42222561792,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.25.ffn_up_exps.weight",
"offset": 42704251392,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.25.ffn_down_shexp.weight",
"offset": 43185940992,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.25.ffn_gate_shexp.weight",
"offset": 43246152192,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.25.ffn_up_shexp.weight",
"offset": 43306363392,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.25.ffn_gate_inp_shexp.weight",
"offset": 43366574592,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 43366588928,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.ffn_gate_inp.weight",
"offset": 43366603264,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.attn_k.bias",
"offset": 43367520768,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 43367522816,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.26.attn_output.weight",
"offset": 43369028096,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.26.attn_q.bias",
"offset": 43379565056,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 43379579392,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.26.attn_v.bias",
"offset": 43390116352,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.26.attn_v.weight",
"offset": 43390118400,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 43391623680,
"shape": [
3584
],
"size": 1505280,
"type": 0
},
{
"name": "blk.26.ffn_down_exps.weight",
"offset": 43391638016,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.26.ffn_gate_exps.weight",
"offset": 43873327616,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.26.ffn_up_exps.weight",
"offset": 44355017216,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.26.ffn_down_shexp.weight",
"offset": 44836706816,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.26.ffn_gate_shexp.weight",
"offset": 44896918016,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.26.ffn_up_shexp.weight",
"offset": 44957129216,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.26.ffn_gate_inp_shexp.weight",
"offset": 45017340416,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 45017354752,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.ffn_gate_inp.weight",
"offset": 45017369088,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.attn_k.bias",
"offset": 45018286592,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 45018288640,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 45019793920,
"shape": [
3584,
3584
],
"size": 1505280,
"type": 14
},
{
"name": "blk.27.attn_q.bias",
"offset": 45030330880,
"shape": [
3584
],
"size": 10536960,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 45030345216,
"shape": [
3584,
3584
],
"size": 14336,
"type": 14
},
{
"name": "blk.27.attn_v.bias",
"offset": 45040882176,
"shape": [
512
],
"size": 10536960,
"type": 0
},
{
"name": "blk.27.attn_v.weight",
"offset": 45040884224,
"shape": [
3584,
512
],
"size": 2048,
"type": 14
},
{
"name": "output.weight",
"offset": 45042389504,
"shape": [
3584,
151936
],
"size": 1505280,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 45489081344,
"shape": [
3584
],
"size": 446691840,
"type": 0
},
{
"name": "blk.27.ffn_down_exps.weight",
"offset": 45489095680,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 14
},
{
"name": "blk.27.ffn_gate_exps.weight",
"offset": 45970785280,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.27.ffn_up_exps.weight",
"offset": 46452474880,
"shape": [
3584,
2560,
64
],
"size": 481689600,
"type": 14
},
{
"name": "blk.27.ffn_down_shexp.weight",
"offset": 46934164480,
"shape": [
20480,
3584
],
"size": 481689600,
"type": 14
},
{
"name": "blk.27.ffn_gate_shexp.weight",
"offset": 46994375680,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.27.ffn_up_shexp.weight",
"offset": 47054586880,
"shape": [
3584,
20480
],
"size": 60211200,
"type": 14
},
{
"name": "blk.27.ffn_gate_inp_shexp.weight",
"offset": 47114798080,
"shape": [
3584
],
"size": 60211200,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 47114812416,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 47114826752,
"shape": [
3584
],
"size": 14336,
"type": 0
}
],
"version": 3
}