Qwen2 MOE 57B
298 Pulls Updated 3 weeks ago
c165271d7cbb · 61GB
{
"metadata": {
"general.architecture": "qwen2moe",
"general.file_type": 7,
"general.name": "Qwen2-57B-A14B-Instruct",
"general.quantization_version": 2,
"qwen2moe.attention.head_count": 28,
"qwen2moe.attention.head_count_kv": 4,
"qwen2moe.attention.layer_norm_rms_epsilon": 0.000001,
"qwen2moe.block_count": 28,
"qwen2moe.context_length": 32768,
"qwen2moe.embedding_length": 3584,
"qwen2moe.expert_count": 64,
"qwen2moe.expert_used_count": 8,
"qwen2moe.feed_forward_length": 20480,
"qwen2moe.rope.freq_base": 1000000,
"split.count": 0,
"split.no": 0,
"split.tensors.count": 479,
"tokenizer.ggml.bos_token_id": 151643,
"tokenizer.ggml.eos_token_id": 151645,
"tokenizer.ggml.merges": "... (151387 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 151643,
"tokenizer.ggml.pre": "qwen2",
"tokenizer.ggml.token_type": "... (151936 values)",
"tokenizer.ggml.tokens": "... (151936 values)"
},
"num_params": 57408658944,
"tensors": [
{
"name": "blk.0.ffn_gate_inp.weight",
"offset": 578572288,
"shape": [
3584,
64
],
"size": 578572288,
"type": 0
},
{
"name": "blk.0.attn_k.bias",
"offset": 579489792,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 579491840,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.0.attn_output.weight",
"offset": 581441536,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.0.attn_q.bias",
"offset": 595089408,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.0.attn_q.weight",
"offset": 595103744,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.0.attn_v.bias",
"offset": 608751616,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.0.attn_v.weight",
"offset": 608753664,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.0.attn_norm.weight",
"offset": 610703360,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.0.ffn_down_exps.weight",
"offset": 610717696,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.0.ffn_gate_exps.weight",
"offset": 1234620416,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.0.ffn_up_exps.weight",
"offset": 1858523136,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.0.ffn_down_shexp.weight",
"offset": 2482425856,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.0.ffn_gate_shexp.weight",
"offset": 2560413696,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.0.ffn_up_shexp.weight",
"offset": 2638401536,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.0.ffn_gate_inp_shexp.weight",
"offset": 2716389376,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 2716403712,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.ffn_gate_inp.weight",
"offset": 2716418048,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.1.attn_k.bias",
"offset": 2717335552,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 2717337600,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.1.attn_output.weight",
"offset": 2719287296,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.1.attn_q.bias",
"offset": 2732935168,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 2732949504,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.1.attn_v.bias",
"offset": 2746597376,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.1.attn_v.weight",
"offset": 2746599424,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.1.attn_norm.weight",
"offset": 2748549120,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.1.ffn_down_exps.weight",
"offset": 2748563456,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.1.ffn_gate_exps.weight",
"offset": 3372466176,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.1.ffn_up_exps.weight",
"offset": 3996368896,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.1.ffn_down_shexp.weight",
"offset": 4620271616,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.1.ffn_gate_shexp.weight",
"offset": 4698259456,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.1.ffn_up_shexp.weight",
"offset": 4776247296,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.1.ffn_gate_inp_shexp.weight",
"offset": 4854235136,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 4854249472,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.ffn_gate_inp.weight",
"offset": 4854263808,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.2.attn_k.bias",
"offset": 4855181312,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 4855183360,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.2.attn_output.weight",
"offset": 4857133056,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.2.attn_q.bias",
"offset": 4870780928,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 4870795264,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.2.attn_v.bias",
"offset": 4884443136,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.2.attn_v.weight",
"offset": 4884445184,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.2.attn_norm.weight",
"offset": 4886394880,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.2.ffn_down_exps.weight",
"offset": 4886409216,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.2.ffn_gate_exps.weight",
"offset": 5510311936,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.2.ffn_up_exps.weight",
"offset": 6134214656,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.2.ffn_down_shexp.weight",
"offset": 6758117376,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.2.ffn_gate_shexp.weight",
"offset": 6836105216,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.2.ffn_up_shexp.weight",
"offset": 6914093056,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.2.ffn_gate_inp_shexp.weight",
"offset": 6992080896,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 6992095232,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.ffn_gate_inp.weight",
"offset": 6992109568,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.3.attn_k.bias",
"offset": 6993027072,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 6993029120,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.3.attn_output.weight",
"offset": 6994978816,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.3.attn_q.bias",
"offset": 7008626688,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 7008641024,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.3.attn_v.bias",
"offset": 7022288896,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.3.attn_v.weight",
"offset": 7022290944,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.3.attn_norm.weight",
"offset": 7024240640,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.3.ffn_down_exps.weight",
"offset": 7024254976,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.3.ffn_gate_exps.weight",
"offset": 7648157696,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.3.ffn_up_exps.weight",
"offset": 8272060416,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.3.ffn_down_shexp.weight",
"offset": 8895963136,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.3.ffn_gate_shexp.weight",
"offset": 8973950976,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.3.ffn_up_shexp.weight",
"offset": 9051938816,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.3.ffn_gate_inp_shexp.weight",
"offset": 9129926656,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 9129940992,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.ffn_gate_inp.weight",
"offset": 9129955328,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.4.attn_k.bias",
"offset": 9130872832,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 9130874880,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.4.attn_output.weight",
"offset": 9132824576,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.4.attn_q.bias",
"offset": 9146472448,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 9146486784,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.4.attn_v.bias",
"offset": 9160134656,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.4.attn_v.weight",
"offset": 9160136704,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.4.attn_norm.weight",
"offset": 9162086400,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.4.ffn_down_exps.weight",
"offset": 9162100736,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.4.ffn_gate_exps.weight",
"offset": 9786003456,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.4.ffn_up_exps.weight",
"offset": 10409906176,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.4.ffn_down_shexp.weight",
"offset": 11033808896,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.4.ffn_gate_shexp.weight",
"offset": 11111796736,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.4.ffn_up_shexp.weight",
"offset": 11189784576,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.4.ffn_gate_inp_shexp.weight",
"offset": 11267772416,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 11267786752,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.ffn_gate_inp.weight",
"offset": 11267801088,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.5.attn_k.bias",
"offset": 11268718592,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 11268720640,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.5.attn_output.weight",
"offset": 11270670336,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.5.attn_q.bias",
"offset": 11284318208,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 11284332544,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.5.attn_v.bias",
"offset": 11297980416,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.5.attn_v.weight",
"offset": 11297982464,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.5.attn_norm.weight",
"offset": 11299932160,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.5.ffn_down_exps.weight",
"offset": 11299946496,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.5.ffn_gate_exps.weight",
"offset": 11923849216,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.5.ffn_up_exps.weight",
"offset": 12547751936,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.5.ffn_down_shexp.weight",
"offset": 13171654656,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.5.ffn_gate_shexp.weight",
"offset": 13249642496,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.5.ffn_up_shexp.weight",
"offset": 13327630336,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.5.ffn_gate_inp_shexp.weight",
"offset": 13405618176,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 13405632512,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.ffn_gate_inp.weight",
"offset": 13405646848,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.6.attn_k.bias",
"offset": 13406564352,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 13406566400,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.6.attn_output.weight",
"offset": 13408516096,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.6.attn_q.bias",
"offset": 13422163968,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 13422178304,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.6.attn_v.bias",
"offset": 13435826176,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.6.attn_v.weight",
"offset": 13435828224,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.6.attn_norm.weight",
"offset": 13437777920,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.6.ffn_down_exps.weight",
"offset": 13437792256,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.6.ffn_gate_exps.weight",
"offset": 14061694976,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.6.ffn_up_exps.weight",
"offset": 14685597696,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.6.ffn_down_shexp.weight",
"offset": 15309500416,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.6.ffn_gate_shexp.weight",
"offset": 15387488256,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.6.ffn_up_shexp.weight",
"offset": 15465476096,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.6.ffn_gate_inp_shexp.weight",
"offset": 15543463936,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 15543478272,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.ffn_gate_inp.weight",
"offset": 15543492608,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.7.attn_k.bias",
"offset": 15544410112,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 15544412160,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.7.attn_output.weight",
"offset": 15546361856,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.7.attn_q.bias",
"offset": 15560009728,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 15560024064,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.7.attn_v.bias",
"offset": 15573671936,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.7.attn_v.weight",
"offset": 15573673984,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.7.attn_norm.weight",
"offset": 15575623680,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.7.ffn_down_exps.weight",
"offset": 15575638016,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.7.ffn_gate_exps.weight",
"offset": 16199540736,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.7.ffn_up_exps.weight",
"offset": 16823443456,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.7.ffn_down_shexp.weight",
"offset": 17447346176,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.7.ffn_gate_shexp.weight",
"offset": 17525334016,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.7.ffn_up_shexp.weight",
"offset": 17603321856,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.7.ffn_gate_inp_shexp.weight",
"offset": 17681309696,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 17681324032,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.ffn_gate_inp.weight",
"offset": 17681338368,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.8.attn_k.bias",
"offset": 17682255872,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 17682257920,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.8.attn_output.weight",
"offset": 17684207616,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.8.attn_q.bias",
"offset": 17697855488,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 17697869824,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.8.attn_v.bias",
"offset": 17711517696,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.8.attn_v.weight",
"offset": 17711519744,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.8.attn_norm.weight",
"offset": 17713469440,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.8.ffn_down_exps.weight",
"offset": 17713483776,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.8.ffn_gate_exps.weight",
"offset": 18337386496,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.8.ffn_up_exps.weight",
"offset": 18961289216,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.8.ffn_down_shexp.weight",
"offset": 19585191936,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.8.ffn_gate_shexp.weight",
"offset": 19663179776,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.8.ffn_up_shexp.weight",
"offset": 19741167616,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.8.ffn_gate_inp_shexp.weight",
"offset": 19819155456,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 19819169792,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.ffn_gate_inp.weight",
"offset": 19819184128,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.9.attn_k.bias",
"offset": 19820101632,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 19820103680,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.9.attn_output.weight",
"offset": 19822053376,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.9.attn_q.bias",
"offset": 19835701248,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 19835715584,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.9.attn_v.bias",
"offset": 19849363456,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.9.attn_v.weight",
"offset": 19849365504,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.10.ffn_gate_inp.weight",
"offset": 19851315200,
"shape": [
3584,
64
],
"size": 1949696,
"type": 0
},
{
"name": "blk.10.attn_k.bias",
"offset": 19852232704,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 19852234752,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.10.attn_output.weight",
"offset": 19854184448,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.10.attn_q.bias",
"offset": 19867832320,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 19867846656,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.10.attn_v.bias",
"offset": 19881494528,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.10.attn_v.weight",
"offset": 19881496576,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.9.attn_norm.weight",
"offset": 19883446272,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.9.ffn_down_exps.weight",
"offset": 19883460608,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.9.ffn_gate_exps.weight",
"offset": 20507363328,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.9.ffn_up_exps.weight",
"offset": 21131266048,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.9.ffn_down_shexp.weight",
"offset": 21755168768,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.9.ffn_gate_shexp.weight",
"offset": 21833156608,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.9.ffn_up_shexp.weight",
"offset": 21911144448,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.9.ffn_gate_inp_shexp.weight",
"offset": 21989132288,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 21989146624,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.attn_norm.weight",
"offset": 21989160960,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.10.ffn_down_exps.weight",
"offset": 21989175296,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.10.ffn_gate_exps.weight",
"offset": 22613078016,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.10.ffn_up_exps.weight",
"offset": 23236980736,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.10.ffn_down_shexp.weight",
"offset": 23860883456,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.10.ffn_gate_shexp.weight",
"offset": 23938871296,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.10.ffn_up_shexp.weight",
"offset": 24016859136,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.10.ffn_gate_inp_shexp.weight",
"offset": 24094846976,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 24094861312,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.ffn_gate_inp.weight",
"offset": 24094875648,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.11.attn_k.bias",
"offset": 24095793152,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 24095795200,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.11.attn_output.weight",
"offset": 24097744896,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.11.attn_q.bias",
"offset": 24111392768,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 24111407104,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.11.attn_v.bias",
"offset": 24125054976,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.11.attn_v.weight",
"offset": 24125057024,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.11.attn_norm.weight",
"offset": 24127006720,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.11.ffn_down_exps.weight",
"offset": 24127021056,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.11.ffn_gate_exps.weight",
"offset": 24750923776,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.11.ffn_up_exps.weight",
"offset": 25374826496,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.11.ffn_down_shexp.weight",
"offset": 25998729216,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.11.ffn_gate_shexp.weight",
"offset": 26076717056,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.11.ffn_up_shexp.weight",
"offset": 26154704896,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.11.ffn_gate_inp_shexp.weight",
"offset": 26232692736,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 26232707072,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.ffn_gate_inp.weight",
"offset": 26232721408,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.12.attn_k.bias",
"offset": 26233638912,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 26233640960,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.12.attn_output.weight",
"offset": 26235590656,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.12.attn_q.bias",
"offset": 26249238528,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 26249252864,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.12.attn_v.bias",
"offset": 26262900736,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.12.attn_v.weight",
"offset": 26262902784,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.12.attn_norm.weight",
"offset": 26264852480,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.12.ffn_down_exps.weight",
"offset": 26264866816,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.12.ffn_gate_exps.weight",
"offset": 26888769536,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.12.ffn_up_exps.weight",
"offset": 27512672256,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.12.ffn_down_shexp.weight",
"offset": 28136574976,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.12.ffn_gate_shexp.weight",
"offset": 28214562816,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.12.ffn_up_shexp.weight",
"offset": 28292550656,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.12.ffn_gate_inp_shexp.weight",
"offset": 28370538496,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 28370552832,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.ffn_gate_inp.weight",
"offset": 28370567168,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.13.attn_k.bias",
"offset": 28371484672,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 28371486720,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.13.attn_output.weight",
"offset": 28373436416,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.13.attn_q.bias",
"offset": 28387084288,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 28387098624,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.13.attn_v.bias",
"offset": 28400746496,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.13.attn_v.weight",
"offset": 28400748544,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.13.attn_norm.weight",
"offset": 28402698240,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.13.ffn_down_exps.weight",
"offset": 28402712576,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.13.ffn_gate_exps.weight",
"offset": 29026615296,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.13.ffn_up_exps.weight",
"offset": 29650518016,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.13.ffn_down_shexp.weight",
"offset": 30274420736,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.13.ffn_gate_shexp.weight",
"offset": 30352408576,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.13.ffn_up_shexp.weight",
"offset": 30430396416,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.13.ffn_gate_inp_shexp.weight",
"offset": 30508384256,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 30508398592,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.ffn_gate_inp.weight",
"offset": 30508412928,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.14.attn_k.bias",
"offset": 30509330432,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 30509332480,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.14.attn_output.weight",
"offset": 30511282176,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.14.attn_q.bias",
"offset": 30524930048,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 30524944384,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.14.attn_v.bias",
"offset": 30538592256,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.14.attn_v.weight",
"offset": 30538594304,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.14.attn_norm.weight",
"offset": 30540544000,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.14.ffn_down_exps.weight",
"offset": 30540558336,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.14.ffn_gate_exps.weight",
"offset": 31164461056,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.14.ffn_up_exps.weight",
"offset": 31788363776,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.14.ffn_down_shexp.weight",
"offset": 32412266496,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.14.ffn_gate_shexp.weight",
"offset": 32490254336,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.14.ffn_up_shexp.weight",
"offset": 32568242176,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.14.ffn_gate_inp_shexp.weight",
"offset": 32646230016,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 32646244352,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.ffn_gate_inp.weight",
"offset": 32646258688,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.15.attn_k.bias",
"offset": 32647176192,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 32647178240,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.15.attn_output.weight",
"offset": 32649127936,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.15.attn_q.bias",
"offset": 32662775808,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 32662790144,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.15.attn_v.bias",
"offset": 32676438016,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.15.attn_v.weight",
"offset": 32676440064,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.15.attn_norm.weight",
"offset": 32678389760,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.15.ffn_down_exps.weight",
"offset": 32678404096,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.15.ffn_gate_exps.weight",
"offset": 33302306816,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.15.ffn_up_exps.weight",
"offset": 33926209536,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.15.ffn_down_shexp.weight",
"offset": 34550112256,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.15.ffn_gate_shexp.weight",
"offset": 34628100096,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.15.ffn_up_shexp.weight",
"offset": 34706087936,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.15.ffn_gate_inp_shexp.weight",
"offset": 34784075776,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 34784090112,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.ffn_gate_inp.weight",
"offset": 34784104448,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.16.attn_k.bias",
"offset": 34785021952,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 34785024000,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.16.attn_output.weight",
"offset": 34786973696,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.16.attn_q.bias",
"offset": 34800621568,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 34800635904,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.16.attn_v.bias",
"offset": 34814283776,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.16.attn_v.weight",
"offset": 34814285824,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.16.attn_norm.weight",
"offset": 34816235520,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.16.ffn_down_exps.weight",
"offset": 34816249856,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.16.ffn_gate_exps.weight",
"offset": 35440152576,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.16.ffn_up_exps.weight",
"offset": 36064055296,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.16.ffn_down_shexp.weight",
"offset": 36687958016,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.16.ffn_gate_shexp.weight",
"offset": 36765945856,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.16.ffn_up_shexp.weight",
"offset": 36843933696,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.16.ffn_gate_inp_shexp.weight",
"offset": 36921921536,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 36921935872,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.ffn_gate_inp.weight",
"offset": 36921950208,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.17.attn_k.bias",
"offset": 36922867712,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 36922869760,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.17.attn_output.weight",
"offset": 36924819456,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.17.attn_q.bias",
"offset": 36938467328,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 36938481664,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.17.attn_v.bias",
"offset": 36952129536,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.17.attn_v.weight",
"offset": 36952131584,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.17.attn_norm.weight",
"offset": 36954081280,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.17.ffn_down_exps.weight",
"offset": 36954095616,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.17.ffn_gate_exps.weight",
"offset": 37577998336,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.17.ffn_up_exps.weight",
"offset": 38201901056,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.17.ffn_down_shexp.weight",
"offset": 38825803776,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.17.ffn_gate_shexp.weight",
"offset": 38903791616,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.17.ffn_up_shexp.weight",
"offset": 38981779456,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.17.ffn_gate_inp_shexp.weight",
"offset": 39059767296,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 39059781632,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.ffn_gate_inp.weight",
"offset": 39059795968,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.18.attn_k.bias",
"offset": 39060713472,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 39060715520,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.18.attn_output.weight",
"offset": 39062665216,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.18.attn_q.bias",
"offset": 39076313088,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 39076327424,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.18.attn_v.bias",
"offset": 39089975296,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.18.attn_v.weight",
"offset": 39089977344,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.18.attn_norm.weight",
"offset": 39091927040,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.18.ffn_down_exps.weight",
"offset": 39091941376,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.18.ffn_gate_exps.weight",
"offset": 39715844096,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.18.ffn_up_exps.weight",
"offset": 40339746816,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.18.ffn_down_shexp.weight",
"offset": 40963649536,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.18.ffn_gate_shexp.weight",
"offset": 41041637376,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.18.ffn_up_shexp.weight",
"offset": 41119625216,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.18.ffn_gate_inp_shexp.weight",
"offset": 41197613056,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 41197627392,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.ffn_gate_inp.weight",
"offset": 41197641728,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.19.attn_k.bias",
"offset": 41198559232,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 41198561280,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.19.attn_output.weight",
"offset": 41200510976,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.19.attn_q.bias",
"offset": 41214158848,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 41214173184,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.19.attn_v.bias",
"offset": 41227821056,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.19.attn_v.weight",
"offset": 41227823104,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.19.attn_norm.weight",
"offset": 41229772800,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.19.ffn_down_exps.weight",
"offset": 41229787136,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.19.ffn_gate_exps.weight",
"offset": 41853689856,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.19.ffn_up_exps.weight",
"offset": 42477592576,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.19.ffn_down_shexp.weight",
"offset": 43101495296,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.19.ffn_gate_shexp.weight",
"offset": 43179483136,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.19.ffn_up_shexp.weight",
"offset": 43257470976,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.19.ffn_gate_inp_shexp.weight",
"offset": 43335458816,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 43335473152,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.ffn_gate_inp.weight",
"offset": 43335487488,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.20.attn_k.bias",
"offset": 43336404992,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 43336407040,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.20.attn_output.weight",
"offset": 43338356736,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.20.attn_q.bias",
"offset": 43352004608,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 43352018944,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.20.attn_v.bias",
"offset": 43365666816,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.20.attn_v.weight",
"offset": 43365668864,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.20.attn_norm.weight",
"offset": 43367618560,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.20.ffn_down_exps.weight",
"offset": 43367632896,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.20.ffn_gate_exps.weight",
"offset": 43991535616,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.20.ffn_up_exps.weight",
"offset": 44615438336,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.20.ffn_down_shexp.weight",
"offset": 45239341056,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.20.ffn_gate_shexp.weight",
"offset": 45317328896,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.20.ffn_up_shexp.weight",
"offset": 45395316736,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.20.ffn_gate_inp_shexp.weight",
"offset": 45473304576,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 45473318912,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.21.ffn_gate_inp.weight",
"offset": 45473333248,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.21.attn_k.bias",
"offset": 45474250752,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 45474252800,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.21.attn_output.weight",
"offset": 45476202496,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.21.attn_q.bias",
"offset": 45489850368,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 45489864704,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.21.attn_v.bias",
"offset": 45503512576,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.21.attn_v.weight",
"offset": 45503514624,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.21.attn_norm.weight",
"offset": 45505464320,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.21.ffn_down_exps.weight",
"offset": 45505478656,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.21.ffn_gate_exps.weight",
"offset": 46129381376,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.21.ffn_up_exps.weight",
"offset": 46753284096,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.21.ffn_down_shexp.weight",
"offset": 47377186816,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.21.ffn_gate_shexp.weight",
"offset": 47455174656,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.21.ffn_up_shexp.weight",
"offset": 47533162496,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.21.ffn_gate_inp_shexp.weight",
"offset": 47611150336,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 47611164672,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.ffn_gate_inp.weight",
"offset": 47611179008,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.22.attn_k.bias",
"offset": 47612096512,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 47612098560,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.22.attn_output.weight",
"offset": 47614048256,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.22.attn_q.bias",
"offset": 47627696128,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 47627710464,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.22.attn_v.bias",
"offset": 47641358336,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.22.attn_v.weight",
"offset": 47641360384,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.22.attn_norm.weight",
"offset": 47643310080,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.22.ffn_down_exps.weight",
"offset": 47643324416,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.22.ffn_gate_exps.weight",
"offset": 48267227136,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.22.ffn_up_exps.weight",
"offset": 48891129856,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.22.ffn_down_shexp.weight",
"offset": 49515032576,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.22.ffn_gate_shexp.weight",
"offset": 49593020416,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.22.ffn_up_shexp.weight",
"offset": 49671008256,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.22.ffn_gate_inp_shexp.weight",
"offset": 49748996096,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 49749010432,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.ffn_gate_inp.weight",
"offset": 49749024768,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.23.attn_k.bias",
"offset": 49749942272,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 49749944320,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.23.attn_output.weight",
"offset": 49751894016,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.23.attn_q.bias",
"offset": 49765541888,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 49765556224,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.23.attn_v.bias",
"offset": 49779204096,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.23.attn_v.weight",
"offset": 49779206144,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.23.attn_norm.weight",
"offset": 49781155840,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.23.ffn_down_exps.weight",
"offset": 49781170176,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.23.ffn_gate_exps.weight",
"offset": 50405072896,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.23.ffn_up_exps.weight",
"offset": 51028975616,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.23.ffn_down_shexp.weight",
"offset": 51652878336,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.23.ffn_gate_shexp.weight",
"offset": 51730866176,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.23.ffn_up_shexp.weight",
"offset": 51808854016,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.23.ffn_gate_inp_shexp.weight",
"offset": 51886841856,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 51886856192,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.ffn_gate_inp.weight",
"offset": 51886870528,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.24.attn_k.bias",
"offset": 51887788032,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 51887790080,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.24.attn_output.weight",
"offset": 51889739776,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.24.attn_q.bias",
"offset": 51903387648,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 51903401984,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.24.attn_v.bias",
"offset": 51917049856,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.24.attn_v.weight",
"offset": 51917051904,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.24.attn_norm.weight",
"offset": 51919001600,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.24.ffn_down_exps.weight",
"offset": 51919015936,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.24.ffn_gate_exps.weight",
"offset": 52542918656,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.24.ffn_up_exps.weight",
"offset": 53166821376,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.24.ffn_down_shexp.weight",
"offset": 53790724096,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.24.ffn_gate_shexp.weight",
"offset": 53868711936,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.24.ffn_up_shexp.weight",
"offset": 53946699776,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.24.ffn_gate_inp_shexp.weight",
"offset": 54024687616,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 54024701952,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.ffn_gate_inp.weight",
"offset": 54024716288,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.25.attn_k.bias",
"offset": 54025633792,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 54025635840,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.25.attn_output.weight",
"offset": 54027585536,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.25.attn_q.bias",
"offset": 54041233408,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 54041247744,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.25.attn_v.bias",
"offset": 54054895616,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.25.attn_v.weight",
"offset": 54054897664,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.25.attn_norm.weight",
"offset": 54056847360,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.25.ffn_down_exps.weight",
"offset": 54056861696,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.25.ffn_gate_exps.weight",
"offset": 54680764416,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.25.ffn_up_exps.weight",
"offset": 55304667136,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.25.ffn_down_shexp.weight",
"offset": 55928569856,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.25.ffn_gate_shexp.weight",
"offset": 56006557696,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.25.ffn_up_shexp.weight",
"offset": 56084545536,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.25.ffn_gate_inp_shexp.weight",
"offset": 56162533376,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 56162547712,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.ffn_gate_inp.weight",
"offset": 56162562048,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.26.attn_k.bias",
"offset": 56163479552,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 56163481600,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.26.attn_output.weight",
"offset": 56165431296,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.26.attn_q.bias",
"offset": 56179079168,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 56179093504,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.26.attn_v.bias",
"offset": 56192741376,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.26.attn_v.weight",
"offset": 56192743424,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.26.attn_norm.weight",
"offset": 56194693120,
"shape": [
3584
],
"size": 1949696,
"type": 0
},
{
"name": "blk.26.ffn_down_exps.weight",
"offset": 56194707456,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.26.ffn_gate_exps.weight",
"offset": 56818610176,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.26.ffn_up_exps.weight",
"offset": 57442512896,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.26.ffn_down_shexp.weight",
"offset": 58066415616,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.26.ffn_gate_shexp.weight",
"offset": 58144403456,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.26.ffn_up_shexp.weight",
"offset": 58222391296,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.26.ffn_gate_inp_shexp.weight",
"offset": 58300379136,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 58300393472,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.ffn_gate_inp.weight",
"offset": 58300407808,
"shape": [
3584,
64
],
"size": 14336,
"type": 0
},
{
"name": "blk.27.attn_k.bias",
"offset": 58301325312,
"shape": [
512
],
"size": 917504,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 58301327360,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "blk.27.attn_output.weight",
"offset": 58303277056,
"shape": [
3584,
3584
],
"size": 1949696,
"type": 8
},
{
"name": "blk.27.attn_q.bias",
"offset": 58316924928,
"shape": [
3584
],
"size": 13647872,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 58316939264,
"shape": [
3584,
3584
],
"size": 14336,
"type": 8
},
{
"name": "blk.27.attn_v.bias",
"offset": 58330587136,
"shape": [
512
],
"size": 13647872,
"type": 0
},
{
"name": "blk.27.attn_v.weight",
"offset": 58330589184,
"shape": [
3584,
512
],
"size": 2048,
"type": 8
},
{
"name": "output.weight",
"offset": 58332538880,
"shape": [
3584,
151936
],
"size": 1949696,
"type": 8
},
{
"name": "blk.27.attn_norm.weight",
"offset": 58911111168,
"shape": [
3584
],
"size": 578572288,
"type": 0
},
{
"name": "blk.27.ffn_down_exps.weight",
"offset": 58911125504,
"shape": [
2560,
3584,
64
],
"size": 14336,
"type": 8
},
{
"name": "blk.27.ffn_gate_exps.weight",
"offset": 59535028224,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.27.ffn_up_exps.weight",
"offset": 60158930944,
"shape": [
3584,
2560,
64
],
"size": 623902720,
"type": 8
},
{
"name": "blk.27.ffn_down_shexp.weight",
"offset": 60782833664,
"shape": [
20480,
3584
],
"size": 623902720,
"type": 8
},
{
"name": "blk.27.ffn_gate_shexp.weight",
"offset": 60860821504,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.27.ffn_up_shexp.weight",
"offset": 60938809344,
"shape": [
3584,
20480
],
"size": 77987840,
"type": 8
},
{
"name": "blk.27.ffn_gate_inp_shexp.weight",
"offset": 61016797184,
"shape": [
3584
],
"size": 77987840,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 61016811520,
"shape": [
3584
],
"size": 14336,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 61016825856,
"shape": [
3584
],
"size": 14336,
"type": 0
}
],
"version": 3
}