Advancing Open-source Large Language Models in Medical Domain , in Chinese or mutil-languages.
70B
270 Pulls Updated 5 weeks ago
7821d006f0f4 · 43GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 15,
"general.name": ".",
"general.quantization_version": 2,
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 8192,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.scores": "... (128256 values)",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "blk.0.attn_q.weight",
"offset": 591003648,
"shape": [
8192,
8192
],
"size": 591003648,
"type": 12
},
{
"name": "blk.0.attn_k.weight",
"offset": 628752384,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.0.attn_v.weight",
"offset": 633470976,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.0.attn_output.weight",
"offset": 640352256,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 678100992,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.0.ffn_up.weight",
"offset": 810221568,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.0.ffn_down.weight",
"offset": 942342144,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.0.attn_norm.weight",
"offset": 1135017984,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1135050752,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 1135083520,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.1.attn_k.weight",
"offset": 1172832256,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.1.attn_v.weight",
"offset": 1177550848,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.1.attn_output.weight",
"offset": 1184432128,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 1222180864,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.1.ffn_up.weight",
"offset": 1354301440,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.1.ffn_down.weight",
"offset": 1486422016,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 1679097856,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 1679130624,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 1679163392,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.2.attn_k.weight",
"offset": 1716912128,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.2.attn_v.weight",
"offset": 1721630720,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.2.attn_output.weight",
"offset": 1728512000,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1766260736,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1898381312,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.2.ffn_down.weight",
"offset": 2030501888,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.2.attn_norm.weight",
"offset": 2223177728,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 2223210496,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 2223243264,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.3.attn_k.weight",
"offset": 2260992000,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.3.attn_v.weight",
"offset": 2265710592,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.3.attn_output.weight",
"offset": 2272591872,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 2310340608,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.3.ffn_up.weight",
"offset": 2442461184,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.3.ffn_down.weight",
"offset": 2574581760,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.3.attn_norm.weight",
"offset": 2767257600,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 2767290368,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 2767323136,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.4.attn_k.weight",
"offset": 2805071872,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.4.attn_v.weight",
"offset": 2809790464,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.4.attn_output.weight",
"offset": 2816671744,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 2854420480,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.4.ffn_up.weight",
"offset": 2986541056,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.4.ffn_down.weight",
"offset": 3118661632,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 3311337472,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 3311370240,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 3311403008,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.5.attn_k.weight",
"offset": 3349151744,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.5.attn_v.weight",
"offset": 3353870336,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.5.attn_output.weight",
"offset": 3360751616,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 3398500352,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.5.ffn_up.weight",
"offset": 3530620928,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.5.ffn_down.weight",
"offset": 3662741504,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.5.attn_norm.weight",
"offset": 3855417344,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 3855450112,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 3855482880,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.6.attn_k.weight",
"offset": 3893231616,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.6.attn_v.weight",
"offset": 3897950208,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.6.attn_output.weight",
"offset": 3904831488,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 3942580224,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.6.ffn_up.weight",
"offset": 4074700800,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.6.ffn_down.weight",
"offset": 4206821376,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "\u0000\u0000 \u0000\u0000\u0000\u0000\u0000\u0000\u0000p\u0000\u0000\u0000\u0000\u0000\u0000\f\u0000\u0000\u0000\u0000",
"offset": 4399497216,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 4399529984,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 4399562752,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.7.attn_k.weight",
"offset": 4437311488,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.7.attn_v.weight",
"offset": 4442030080,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.7.attn_output.weight",
"offset": 4448911360,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 4486660096,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.7.ffn_up.weight",
"offset": 4618780672,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.7.ffn_down.weight",
"offset": 4750901248,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 4943577088,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 4943609856,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 4943642624,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.8.attn_k.weight",
"offset": 4981391360,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.8.attn_v.weight",
"offset": 4986109952,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.8.attn_output.weight",
"offset": 4992991232,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 5030739968,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.8.ffn_up.weight",
"offset": 5162860544,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.8.ffn_down.weight",
"offset": 5294981120,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 5487656960,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 5487689728,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 5487722496,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.9.attn_k.weight",
"offset": 5525471232,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.9.attn_v.weight",
"offset": 5530189824,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.9.attn_output.weight",
"offset": 5537071104,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 5574819840,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.9.ffn_up.weight",
"offset": 5706940416,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.9.ffn_down.weight",
"offset": 5839060992,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 6031736832,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 6031769600,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 6031802368,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.10.attn_k.weight",
"offset": 6069551104,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.10.attn_v.weight",
"offset": 6074269696,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 6080036864,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 6117785600,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.10.ffn_up.weight",
"offset": 6249906176,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.10.ffn_down.weight",
"offset": 6382026752,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.10.attn_norm.weight",
"offset": 6514147328,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 6514180096,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 6514212864,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.11.attn_k.weight",
"offset": 6551961600,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.11.attn_v.weight",
"offset": 6556680192,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 6562447360,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 6600196096,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.11.ffn_up.weight",
"offset": 6732316672,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.11.ffn_down.weight",
"offset": 6864437248,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.11.attn_norm.weight",
"offset": 6996557824,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 6996590592,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 6996623360,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.12.attn_k.weight",
"offset": 7034372096,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.12.attn_v.weight",
"offset": 7039090688,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.12.attn_output.weight",
"offset": 7045971968,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 7083720704,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.12.ffn_up.weight",
"offset": 7215841280,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.12.ffn_down.weight",
"offset": 7347961856,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 7540637696,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 7540670464,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 7540703232,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.13.attn_k.weight",
"offset": 7578451968,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.13.attn_v.weight",
"offset": 7583170560,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 7588937728,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 7626686464,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.13.ffn_up.weight",
"offset": 7758807040,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.13.ffn_down.weight",
"offset": 7890927616,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.13.attn_norm.weight",
"offset": 8023048192,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 8023080960,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 8023113728,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.14.attn_k.weight",
"offset": 8060862464,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.14.attn_v.weight",
"offset": 8065581056,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 8071348224,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 8109096960,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.14.ffn_up.weight",
"offset": 8241217536,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.14.ffn_down.weight",
"offset": 8373338112,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.14.attn_norm.weight",
"offset": 8505458688,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 8505491456,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 8505524224,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.15.attn_k.weight",
"offset": 8543272960,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.15.attn_v.weight",
"offset": 8547991552,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.15.attn_output.weight",
"offset": 8554872832,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 8592621568,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.15.ffn_up.weight",
"offset": 8724742144,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.15.ffn_down.weight",
"offset": 8856862720,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 9049538560,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 9049571328,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 9049604096,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.16.attn_k.weight",
"offset": 9087352832,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.16.attn_v.weight",
"offset": 9092071424,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 9097838592,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 9135587328,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 9267707904,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.16.ffn_down.weight",
"offset": 9399828480,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.16.attn_norm.weight",
"offset": 9531949056,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 9531981824,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 9532014592,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.17.attn_k.weight",
"offset": 9569763328,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.17.attn_v.weight",
"offset": 9574481920,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 9580249088,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 9617997824,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.17.ffn_up.weight",
"offset": 9750118400,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.17.ffn_down.weight",
"offset": 9882238976,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.17.attn_norm.weight",
"offset": 10014359552,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 10014392320,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 10014425088,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.18.attn_k.weight",
"offset": 10052173824,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.18.attn_v.weight",
"offset": 10056892416,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.18.attn_output.weight",
"offset": 10063773696,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 10101522432,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.18.ffn_up.weight",
"offset": 10233643008,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.18.ffn_down.weight",
"offset": 10365763584,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 10558439424,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 10558472192,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 10558504960,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.19.attn_k.weight",
"offset": 10596253696,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.19.attn_v.weight",
"offset": 10600972288,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 10606739456,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 10644488192,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.19.ffn_up.weight",
"offset": 10776608768,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.19.ffn_down.weight",
"offset": 10908729344,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.19.attn_norm.weight",
"offset": 11040849920,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 11040882688,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 11040915456,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.20.attn_k.weight",
"offset": 11078664192,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.20.attn_v.weight",
"offset": 11083382784,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 11089149952,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 11126898688,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.20.ffn_up.weight",
"offset": 11259019264,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.20.ffn_down.weight",
"offset": 11391139840,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.20.attn_norm.weight",
"offset": 11523260416,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 11523293184,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 11523325952,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.21.attn_k.weight",
"offset": 11561074688,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.21.attn_v.weight",
"offset": 11565793280,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.21.attn_output.weight",
"offset": 11572674560,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 11610423296,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.21.ffn_up.weight",
"offset": 11742543872,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.21.ffn_down.weight",
"offset": 11874664448,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.21.attn_norm.weight",
"offset": 12067340288,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 12067373056,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 12067405824,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.22.attn_k.weight",
"offset": 12105154560,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.22.attn_v.weight",
"offset": 12109873152,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 12115640320,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 12153389056,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.22.ffn_up.weight",
"offset": 12285509632,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.22.ffn_down.weight",
"offset": 12417630208,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.22.attn_norm.weight",
"offset": 12549750784,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 12549783552,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 12549816320,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.23.attn_k.weight",
"offset": 12587565056,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.23.attn_v.weight",
"offset": 12592283648,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 12598050816,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 12635799552,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.23.ffn_up.weight",
"offset": 12767920128,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.23.ffn_down.weight",
"offset": 12900040704,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.23.attn_norm.weight",
"offset": 13032161280,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 13032194048,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 13032226816,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.24.attn_k.weight",
"offset": 13069975552,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.24.attn_v.weight",
"offset": 13074694144,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.24.attn_output.weight",
"offset": 13081575424,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 13119324160,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.24.ffn_up.weight",
"offset": 13251444736,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.24.ffn_down.weight",
"offset": 13383565312,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.24.attn_norm.weight",
"offset": 13576241152,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 13576273920,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 13576306688,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.25.attn_k.weight",
"offset": 13614055424,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.25.attn_v.weight",
"offset": 13618774016,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 13624541184,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 13662289920,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.25.ffn_up.weight",
"offset": 13794410496,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.25.ffn_down.weight",
"offset": 13926531072,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.25.attn_norm.weight",
"offset": 14058651648,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 14058684416,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 14058717184,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.26.attn_k.weight",
"offset": 14096465920,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.26.attn_v.weight",
"offset": 14101184512,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 14106951680,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 14144700416,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.26.ffn_up.weight",
"offset": 14276820992,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.26.ffn_down.weight",
"offset": 14408941568,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.26.attn_norm.weight",
"offset": 14541062144,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 14541094912,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 14541127680,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.27.attn_k.weight",
"offset": 14578876416,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.27.attn_v.weight",
"offset": 14583595008,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.27.attn_output.weight",
"offset": 14590476288,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 14628225024,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.27.ffn_up.weight",
"offset": 14760345600,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.27.ffn_down.weight",
"offset": 14892466176,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.27.attn_norm.weight",
"offset": 15085142016,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 15085174784,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.28.attn_q.weight",
"offset": 15085207552,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.28.attn_k.weight",
"offset": 15122956288,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.28.attn_v.weight",
"offset": 15127674880,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 15133442048,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 15171190784,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.28.ffn_up.weight",
"offset": 15303311360,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.28.ffn_down.weight",
"offset": 15435431936,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.28.attn_norm.weight",
"offset": 15567552512,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 15567585280,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.29.attn_q.weight",
"offset": 15567618048,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.29.attn_k.weight",
"offset": 15605366784,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.29.attn_v.weight",
"offset": 15610085376,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 15615852544,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 15653601280,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 15785721856,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.29.ffn_down.weight",
"offset": 15917842432,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.29.attn_norm.weight",
"offset": 16049963008,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 16049995776,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.30.attn_q.weight",
"offset": 16050028544,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.30.attn_k.weight",
"offset": 16087777280,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.30.attn_v.weight",
"offset": 16092495872,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.30.attn_output.weight",
"offset": 16099377152,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 16137125888,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.30.ffn_up.weight",
"offset": 16269246464,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.30.ffn_down.weight",
"offset": 16401367040,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.30.attn_norm.weight",
"offset": 16594042880,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 16594075648,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.31.attn_q.weight",
"offset": 16594108416,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.31.attn_k.weight",
"offset": 16631857152,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.31.attn_v.weight",
"offset": 16636575744,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 16642342912,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 16680091648,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.31.ffn_up.weight",
"offset": 16812212224,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.31.ffn_down.weight",
"offset": 16944332800,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.31.attn_norm.weight",
"offset": 17076453376,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 17076486144,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.32.attn_q.weight",
"offset": 17076518912,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.32.attn_k.weight",
"offset": 17114267648,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.32.attn_v.weight",
"offset": 17118986240,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.32.attn_output.weight",
"offset": 17124753408,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 17162502144,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.32.ffn_up.weight",
"offset": 17294622720,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.32.ffn_down.weight",
"offset": 17426743296,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.32.attn_norm.weight",
"offset": 17558863872,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 17558896640,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.33.attn_q.weight",
"offset": 17558929408,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.33.attn_k.weight",
"offset": 17596678144,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.33.attn_v.weight",
"offset": 17601396736,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.33.attn_output.weight",
"offset": 17608278016,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 17646026752,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.33.ffn_up.weight",
"offset": 17778147328,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.33.ffn_down.weight",
"offset": 17910267904,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.33.attn_norm.weight",
"offset": 18102943744,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 18102976512,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.34.attn_q.weight",
"offset": 18103009280,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.34.attn_k.weight",
"offset": 18140758016,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.34.attn_v.weight",
"offset": 18145476608,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.34.attn_output.weight",
"offset": 18151243776,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 18188992512,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.34.ffn_up.weight",
"offset": 18321113088,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.34.ffn_down.weight",
"offset": 18453233664,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.34.attn_norm.weight",
"offset": 18585354240,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 18585387008,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.35.attn_q.weight",
"offset": 18585419776,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.35.attn_k.weight",
"offset": 18623168512,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.35.attn_v.weight",
"offset": 18627887104,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.35.attn_output.weight",
"offset": 18633654272,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 18671403008,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.35.ffn_up.weight",
"offset": 18803523584,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.35.ffn_down.weight",
"offset": 18935644160,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.35.attn_norm.weight",
"offset": 19067764736,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 19067797504,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.36.attn_q.weight",
"offset": 19067830272,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.36.attn_k.weight",
"offset": 19105579008,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.36.attn_v.weight",
"offset": 19110297600,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.36.attn_output.weight",
"offset": 19117178880,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 19154927616,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.36.ffn_up.weight",
"offset": 19287048192,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.36.ffn_down.weight",
"offset": 19419168768,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.36.attn_norm.weight",
"offset": 19611844608,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 19611877376,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.37.attn_q.weight",
"offset": 19611910144,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.37.attn_k.weight",
"offset": 19649658880,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.37.attn_v.weight",
"offset": 19654377472,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.37.attn_output.weight",
"offset": 19660144640,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 19697893376,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.37.ffn_up.weight",
"offset": 19830013952,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.37.ffn_down.weight",
"offset": 19962134528,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.37.attn_norm.weight",
"offset": 20094255104,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 20094287872,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.38.attn_q.weight",
"offset": 20094320640,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.38.attn_k.weight",
"offset": 20132069376,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.38.attn_v.weight",
"offset": 20136787968,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.38.attn_output.weight",
"offset": 20142555136,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 20180303872,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.38.ffn_up.weight",
"offset": 20312424448,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.38.ffn_down.weight",
"offset": 20444545024,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.38.attn_norm.weight",
"offset": 20576665600,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 20576698368,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.39.attn_q.weight",
"offset": 20576731136,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.39.attn_k.weight",
"offset": 20614479872,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.39.attn_v.weight",
"offset": 20619198464,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.39.attn_output.weight",
"offset": 20626079744,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 20663828480,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.39.ffn_up.weight",
"offset": 20795949056,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.39.ffn_down.weight",
"offset": 20928069632,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.39.attn_norm.weight",
"offset": 21120745472,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 21120778240,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.40.attn_q.weight",
"offset": 21120811008,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.40.attn_k.weight",
"offset": 21158559744,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.40.attn_v.weight",
"offset": 21163278336,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.40.attn_output.weight",
"offset": 21169045504,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 21206794240,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.40.ffn_up.weight",
"offset": 21338914816,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.40.ffn_down.weight",
"offset": 21471035392,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.40.attn_norm.weight",
"offset": 21603155968,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 21603188736,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.41.attn_q.weight",
"offset": 21603221504,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.41.attn_k.weight",
"offset": 21640970240,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.41.attn_v.weight",
"offset": 21645688832,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.41.attn_output.weight",
"offset": 21651456000,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 21689204736,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.41.ffn_up.weight",
"offset": 21821325312,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.41.ffn_down.weight",
"offset": 21953445888,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.41.attn_norm.weight",
"offset": 22085566464,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 22085599232,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.42.attn_q.weight",
"offset": 22085632000,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.42.attn_k.weight",
"offset": 22123380736,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.42.attn_v.weight",
"offset": 22128099328,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.42.attn_output.weight",
"offset": 22134980608,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 22172729344,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.42.ffn_up.weight",
"offset": 22304849920,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.42.ffn_down.weight",
"offset": 22436970496,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 22629646336,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 22629679104,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.43.attn_q.weight",
"offset": 22629711872,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.43.attn_k.weight",
"offset": 22667460608,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.43.attn_v.weight",
"offset": 22672179200,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.43.attn_output.weight",
"offset": 22677946368,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 22715695104,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.43.ffn_up.weight",
"offset": 22847815680,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.43.ffn_down.weight",
"offset": 22979936256,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.43.attn_norm.weight",
"offset": 23112056832,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 23112089600,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.44.attn_q.weight",
"offset": 23112122368,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.44.attn_k.weight",
"offset": 23149871104,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.44.attn_v.weight",
"offset": 23154589696,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.44.attn_output.weight",
"offset": 23160356864,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 23198105600,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.44.ffn_up.weight",
"offset": 23330226176,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.44.ffn_down.weight",
"offset": 23462346752,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.44.attn_norm.weight",
"offset": 23594467328,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 23594500096,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.45.attn_q.weight",
"offset": 23594532864,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.45.attn_k.weight",
"offset": 23632281600,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.45.attn_v.weight",
"offset": 23637000192,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.45.attn_output.weight",
"offset": 23643881472,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 23681630208,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.45.ffn_up.weight",
"offset": 23813750784,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.45.ffn_down.weight",
"offset": 23945871360,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 24138547200,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 24138579968,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.46.attn_q.weight",
"offset": 24138612736,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.46.attn_k.weight",
"offset": 24176361472,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.46.attn_v.weight",
"offset": 24181080064,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.46.attn_output.weight",
"offset": 24186847232,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 24224595968,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.46.ffn_up.weight",
"offset": 24356716544,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.46.ffn_down.weight",
"offset": 24488837120,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.46.attn_norm.weight",
"offset": 24620957696,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 24620990464,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.47.attn_q.weight",
"offset": 24621023232,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.47.attn_k.weight",
"offset": 24658771968,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.47.attn_v.weight",
"offset": 24663490560,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.47.attn_output.weight",
"offset": 24669257728,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 24707006464,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.47.ffn_up.weight",
"offset": 24839127040,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.47.ffn_down.weight",
"offset": 24971247616,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.47.attn_norm.weight",
"offset": 25103368192,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 25103400960,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.48.attn_q.weight",
"offset": 25103433728,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.48.attn_k.weight",
"offset": 25141182464,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.48.attn_v.weight",
"offset": 25145901056,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.48.attn_output.weight",
"offset": 25152782336,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 25190531072,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.48.ffn_up.weight",
"offset": 25322651648,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.48.ffn_down.weight",
"offset": 25454772224,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.48.attn_norm.weight",
"offset": 25647448064,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 25647480832,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.49.attn_q.weight",
"offset": 25647513600,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.49.attn_k.weight",
"offset": 25685262336,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.49.attn_v.weight",
"offset": 25689980928,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.49.attn_output.weight",
"offset": 25695748096,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 25733496832,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.49.ffn_up.weight",
"offset": 25865617408,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.49.ffn_down.weight",
"offset": 25997737984,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.49.attn_norm.weight",
"offset": 26129858560,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 26129891328,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.50.attn_q.weight",
"offset": 26129924096,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.50.attn_k.weight",
"offset": 26167672832,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.50.attn_v.weight",
"offset": 26172391424,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.50.attn_output.weight",
"offset": 26178158592,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 26215907328,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.50.ffn_up.weight",
"offset": 26348027904,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.50.ffn_down.weight",
"offset": 26480148480,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.50.attn_norm.weight",
"offset": 26612269056,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 26612301824,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.51.attn_q.weight",
"offset": 26612334592,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.51.attn_k.weight",
"offset": 26650083328,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.51.attn_v.weight",
"offset": 26654801920,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.51.attn_output.weight",
"offset": 26661683200,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 26699431936,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.51.ffn_up.weight",
"offset": 26831552512,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.51.ffn_down.weight",
"offset": 26963673088,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.51.attn_norm.weight",
"offset": 27156348928,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 27156381696,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.52.attn_q.weight",
"offset": 27156414464,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.52.attn_k.weight",
"offset": 27194163200,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.52.attn_v.weight",
"offset": 27198881792,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.52.attn_output.weight",
"offset": 27204648960,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 27242397696,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.52.ffn_up.weight",
"offset": 27374518272,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.52.ffn_down.weight",
"offset": 27506638848,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.52.attn_norm.weight",
"offset": 27638759424,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 27638792192,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.53.attn_q.weight",
"offset": 27638824960,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.53.attn_k.weight",
"offset": 27676573696,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.53.attn_v.weight",
"offset": 27681292288,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.53.attn_output.weight",
"offset": 27687059456,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 27724808192,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.53.ffn_up.weight",
"offset": 27856928768,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.53.ffn_down.weight",
"offset": 27989049344,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.53.attn_norm.weight",
"offset": 28121169920,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 28121202688,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.54.attn_q.weight",
"offset": 28121235456,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.54.attn_k.weight",
"offset": 28158984192,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.54.attn_v.weight",
"offset": 28163702784,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.54.attn_output.weight",
"offset": 28170584064,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 28208332800,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.54.ffn_up.weight",
"offset": 28340453376,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.54.ffn_down.weight",
"offset": 28472573952,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.54.attn_norm.weight",
"offset": 28665249792,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 28665282560,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.55.attn_q.weight",
"offset": 28665315328,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.55.attn_k.weight",
"offset": 28703064064,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.55.attn_v.weight",
"offset": 28707782656,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.55.attn_output.weight",
"offset": 28713549824,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 28751298560,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.55.ffn_up.weight",
"offset": 28883419136,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.55.ffn_down.weight",
"offset": 29015539712,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.55.attn_norm.weight",
"offset": 29147660288,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 29147693056,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.56.attn_q.weight",
"offset": 29147725824,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.56.attn_k.weight",
"offset": 29185474560,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.56.attn_v.weight",
"offset": 29190193152,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.56.attn_output.weight",
"offset": 29195960320,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 29233709056,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.56.ffn_up.weight",
"offset": 29365829632,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.56.ffn_down.weight",
"offset": 29497950208,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.56.attn_norm.weight",
"offset": 29630070784,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 29630103552,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.57.attn_q.weight",
"offset": 29630136320,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.57.attn_k.weight",
"offset": 29667885056,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.57.attn_v.weight",
"offset": 29672603648,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.57.attn_output.weight",
"offset": 29679484928,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 29717233664,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.57.ffn_up.weight",
"offset": 29849354240,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.57.ffn_down.weight",
"offset": 29981474816,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.57.attn_norm.weight",
"offset": 30174150656,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 30174183424,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.58.attn_q.weight",
"offset": 30174216192,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.58.attn_k.weight",
"offset": 30211964928,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.58.attn_v.weight",
"offset": 30216683520,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.58.attn_output.weight",
"offset": 30222450688,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 30260199424,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.58.ffn_up.weight",
"offset": 30392320000,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.58.ffn_down.weight",
"offset": 30524440576,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.58.attn_norm.weight",
"offset": 30656561152,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 30656593920,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.59.attn_q.weight",
"offset": 30656626688,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.59.attn_k.weight",
"offset": 30694375424,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.59.attn_v.weight",
"offset": 30699094016,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.59.attn_output.weight",
"offset": 30704861184,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 30742609920,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.59.ffn_up.weight",
"offset": 30874730496,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.59.ffn_down.weight",
"offset": 31006851072,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.59.attn_norm.weight",
"offset": 31138971648,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 31139004416,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.60.attn_q.weight",
"offset": 31139037184,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.60.attn_k.weight",
"offset": 31176785920,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.60.attn_v.weight",
"offset": 31181504512,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.60.attn_output.weight",
"offset": 31188385792,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 31226134528,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.60.ffn_up.weight",
"offset": 31358255104,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.60.ffn_down.weight",
"offset": 31490375680,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.60.attn_norm.weight",
"offset": 31683051520,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 31683084288,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.61.attn_q.weight",
"offset": 31683117056,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.61.attn_k.weight",
"offset": 31720865792,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.61.attn_v.weight",
"offset": 31725584384,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.61.attn_output.weight",
"offset": 31731351552,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 31769100288,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.61.ffn_up.weight",
"offset": 31901220864,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.61.ffn_down.weight",
"offset": 32033341440,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.61.attn_norm.weight",
"offset": 32165462016,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 32165494784,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.62.attn_q.weight",
"offset": 32165527552,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.62.attn_k.weight",
"offset": 32203276288,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.62.attn_v.weight",
"offset": 32207994880,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.62.attn_output.weight",
"offset": 32213762048,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 32251510784,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.62.ffn_up.weight",
"offset": 32383631360,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.62.ffn_down.weight",
"offset": 32515751936,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.62.attn_norm.weight",
"offset": 32647872512,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 32647905280,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.63.attn_q.weight",
"offset": 32647938048,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.63.attn_k.weight",
"offset": 32685686784,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.63.attn_v.weight",
"offset": 32690405376,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.63.attn_output.weight",
"offset": 32697286656,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 32735035392,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.63.ffn_up.weight",
"offset": 32867155968,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.63.ffn_down.weight",
"offset": 32999276544,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.63.attn_norm.weight",
"offset": 33191952384,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 33191985152,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.64.attn_q.weight",
"offset": 33192017920,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.64.attn_k.weight",
"offset": 33229766656,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.64.attn_v.weight",
"offset": 33234485248,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.64.attn_output.weight",
"offset": 33240252416,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 33278001152,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.64.ffn_up.weight",
"offset": 33410121728,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.64.ffn_down.weight",
"offset": 33542242304,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.64.attn_norm.weight",
"offset": 33674362880,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 33674395648,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.65.attn_q.weight",
"offset": 33674428416,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.65.attn_k.weight",
"offset": 33712177152,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.65.attn_v.weight",
"offset": 33716895744,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.65.attn_output.weight",
"offset": 33722662912,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 33760411648,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.65.ffn_up.weight",
"offset": 33892532224,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.65.ffn_down.weight",
"offset": 34024652800,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.65.attn_norm.weight",
"offset": 34156773376,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 34156806144,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.66.attn_q.weight",
"offset": 34156838912,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.66.attn_k.weight",
"offset": 34194587648,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.66.attn_v.weight",
"offset": 34199306240,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.66.attn_output.weight",
"offset": 34206187520,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 34243936256,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.66.ffn_up.weight",
"offset": 34376056832,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.66.ffn_down.weight",
"offset": 34508177408,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.66.attn_norm.weight",
"offset": 34700853248,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 34700886016,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.67.attn_q.weight",
"offset": 34700918784,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.67.attn_k.weight",
"offset": 34738667520,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.67.attn_v.weight",
"offset": 34743386112,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.67.attn_output.weight",
"offset": 34749153280,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 34786902016,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.67.ffn_up.weight",
"offset": 34919022592,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.67.ffn_down.weight",
"offset": 35051143168,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.67.attn_norm.weight",
"offset": 35183263744,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 35183296512,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.68.attn_q.weight",
"offset": 35183329280,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.68.attn_k.weight",
"offset": 35221078016,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.68.attn_v.weight",
"offset": 35225796608,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 13
},
{
"name": "blk.68.attn_output.weight",
"offset": 35231563776,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 12
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 35269312512,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.68.ffn_up.weight",
"offset": 35401433088,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.68.ffn_down.weight",
"offset": 35533553664,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 12
},
{
"name": "blk.68.attn_norm.weight",
"offset": 35665674240,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 35665707008,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.69.attn_q.weight",
"offset": 35665739776,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.69.attn_k.weight",
"offset": 35703488512,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.69.attn_v.weight",
"offset": 35708207104,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.69.attn_output.weight",
"offset": 35715088384,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 35752837120,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.69.ffn_up.weight",
"offset": 35884957696,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.69.ffn_down.weight",
"offset": 36017078272,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.69.attn_norm.weight",
"offset": 36209754112,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 36209786880,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.70.attn_q.weight",
"offset": 36209819648,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.70.attn_k.weight",
"offset": 36247568384,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.70.attn_v.weight",
"offset": 36252286976,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.70.attn_output.weight",
"offset": 36259168256,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 36296916992,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.70.ffn_up.weight",
"offset": 36429037568,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.70.ffn_down.weight",
"offset": 36561158144,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.70.attn_norm.weight",
"offset": 36753833984,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 36753866752,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.71.attn_q.weight",
"offset": 36753899520,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.71.attn_k.weight",
"offset": 36791648256,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.71.attn_v.weight",
"offset": 36796366848,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.71.attn_output.weight",
"offset": 36803248128,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 36840996864,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.71.ffn_up.weight",
"offset": 36973117440,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.71.ffn_down.weight",
"offset": 37105238016,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.71.attn_norm.weight",
"offset": 37297913856,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 37297946624,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.72.attn_q.weight",
"offset": 37297979392,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.72.attn_k.weight",
"offset": 37335728128,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.72.attn_v.weight",
"offset": 37340446720,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.72.attn_output.weight",
"offset": 37347328000,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 37385076736,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.72.ffn_up.weight",
"offset": 37517197312,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.72.ffn_down.weight",
"offset": 37649317888,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.72.attn_norm.weight",
"offset": 37841993728,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 37842026496,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.73.attn_q.weight",
"offset": 37842059264,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.73.attn_k.weight",
"offset": 37879808000,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.73.attn_v.weight",
"offset": 37884526592,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.73.attn_output.weight",
"offset": 37891407872,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 37929156608,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.73.ffn_up.weight",
"offset": 38061277184,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.73.ffn_down.weight",
"offset": 38193397760,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.73.attn_norm.weight",
"offset": 38386073600,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 38386106368,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.74.attn_q.weight",
"offset": 38386139136,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.74.attn_k.weight",
"offset": 38423887872,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.74.attn_v.weight",
"offset": 38428606464,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.74.attn_output.weight",
"offset": 38435487744,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 38473236480,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.74.ffn_up.weight",
"offset": 38605357056,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.74.ffn_down.weight",
"offset": 38737477632,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.74.attn_norm.weight",
"offset": 38930153472,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 38930186240,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.75.attn_q.weight",
"offset": 38930219008,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.75.attn_k.weight",
"offset": 38967967744,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.75.attn_v.weight",
"offset": 38972686336,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.75.attn_output.weight",
"offset": 38979567616,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 39017316352,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.75.ffn_up.weight",
"offset": 39149436928,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.75.ffn_down.weight",
"offset": 39281557504,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.75.attn_norm.weight",
"offset": 39474233344,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 39474266112,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.76.attn_q.weight",
"offset": 39474298880,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.76.attn_k.weight",
"offset": 39512047616,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.76.attn_v.weight",
"offset": 39516766208,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.76.attn_output.weight",
"offset": 39523647488,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 39561396224,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.76.ffn_up.weight",
"offset": 39693516800,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.76.ffn_down.weight",
"offset": 39825637376,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.76.attn_norm.weight",
"offset": 40018313216,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 40018345984,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.77.attn_q.weight",
"offset": 40018378752,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.77.attn_k.weight",
"offset": 40056127488,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.77.attn_v.weight",
"offset": 40060846080,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.77.attn_output.weight",
"offset": 40067727360,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 40105476096,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.77.ffn_up.weight",
"offset": 40237596672,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.77.ffn_down.weight",
"offset": 40369717248,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.77.attn_norm.weight",
"offset": 40562393088,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 40562425856,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.78.attn_q.weight",
"offset": 40562458624,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.78.attn_k.weight",
"offset": 40600207360,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.78.attn_v.weight",
"offset": 40604925952,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.78.attn_output.weight",
"offset": 40611807232,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 40649555968,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.78.ffn_up.weight",
"offset": 40781676544,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.78.ffn_down.weight",
"offset": 40913797120,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.78.attn_norm.weight",
"offset": 41106472960,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 41106505728,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.79.attn_q.weight",
"offset": 41106538496,
"shape": [
8192,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.79.attn_k.weight",
"offset": 41144287232,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 12
},
{
"name": "blk.79.attn_v.weight",
"offset": 41149005824,
"shape": [
8192,
1024
],
"size": 4718592,
"type": 14
},
{
"name": "blk.79.attn_output.weight",
"offset": 41155887104,
"shape": [
8192,
8192
],
"size": 6881280,
"type": 12
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 41193635840,
"shape": [
8192,
28672
],
"size": 37748736,
"type": 12
},
{
"name": "blk.79.ffn_up.weight",
"offset": 41325756416,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.79.ffn_down.weight",
"offset": 41457876992,
"shape": [
28672,
8192
],
"size": 132120576,
"type": 14
},
{
"name": "blk.79.attn_norm.weight",
"offset": 41650552832,
"shape": [
8192
],
"size": 192675840,
"type": 0
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 41650585600,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 41650618368,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output.weight",
"offset": 41650651136,
"shape": [
8192,
128256
],
"size": 32768,
"type": 14
}
],
"version": 3
}