Advancing Open-source Large Language Models in Medical Domain
8B
70B
477 Pulls Updated 5 weeks ago
39b5293b2a94 · 26GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 10,
"general.name": ".",
"general.quantization_version": 2,
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 8192,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 500000,
"llama.vocab_size": 128256,
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.padding_token_id": 128001,
"tokenizer.ggml.scores": "... (128256 values)",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "blk.0.attn_q.weight",
"offset": 344752128,
"shape": [
8192,
8192
],
"size": 344752128,
"type": 10
},
{
"name": "blk.0.attn_k.weight",
"offset": 366772224,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.0.attn_v.weight",
"offset": 369524736,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.0.attn_output.weight",
"offset": 375291904,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 404127744,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.0.ffn_up.weight",
"offset": 481198080,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.0.ffn_down.weight",
"offset": 558268416,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.0.attn_norm.weight",
"offset": 659193856,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 659226624,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.1.attn_q.weight",
"offset": 659259392,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.1.attn_k.weight",
"offset": 681279488,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.1.attn_v.weight",
"offset": 684032000,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.1.attn_output.weight",
"offset": 689799168,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 718635008,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.1.ffn_up.weight",
"offset": 795705344,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.1.ffn_down.weight",
"offset": 872775680,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.1.attn_norm.weight",
"offset": 973701120,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 973733888,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.2.attn_q.weight",
"offset": 973766656,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.2.attn_k.weight",
"offset": 995786752,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.2.attn_v.weight",
"offset": 998539264,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.2.attn_output.weight",
"offset": 1004306432,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 1033142272,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.2.ffn_up.weight",
"offset": 1110212608,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.2.ffn_down.weight",
"offset": 1187282944,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.2.attn_norm.weight",
"offset": 1288208384,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 1288241152,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.3.attn_q.weight",
"offset": 1288273920,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.3.attn_k.weight",
"offset": 1310294016,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.3.attn_v.weight",
"offset": 1313046528,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.3.attn_output.weight",
"offset": 1318813696,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 1347649536,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.3.ffn_up.weight",
"offset": 1424719872,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.3.ffn_down.weight",
"offset": 1501790208,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.3.attn_norm.weight",
"offset": 1602715648,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 1602748416,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.4.attn_q.weight",
"offset": 1602781184,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.4.attn_k.weight",
"offset": 1624801280,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.4.attn_v.weight",
"offset": 1627553792,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.4.attn_output.weight",
"offset": 1633320960,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 1662156800,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.4.ffn_up.weight",
"offset": 1739227136,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.4.ffn_down.weight",
"offset": 1816297472,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.4.attn_norm.weight",
"offset": 1917222912,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 1917255680,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.5.attn_q.weight",
"offset": 1917288448,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.5.attn_k.weight",
"offset": 1939308544,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.5.attn_v.weight",
"offset": 1942061056,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.5.attn_output.weight",
"offset": 1947828224,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 1976664064,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.5.ffn_up.weight",
"offset": 2053734400,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.5.ffn_down.weight",
"offset": 2130804736,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.5.attn_norm.weight",
"offset": 2231730176,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 2231762944,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.6.attn_q.weight",
"offset": 2231795712,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.6.attn_k.weight",
"offset": 2253815808,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.6.attn_v.weight",
"offset": 2256568320,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.6.attn_output.weight",
"offset": 2262335488,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 2291171328,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.6.ffn_up.weight",
"offset": 2368241664,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.6.ffn_down.weight",
"offset": 2445312000,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.6.attn_norm.weight",
"offset": 2546237440,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 2546270208,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.7.attn_q.weight",
"offset": 2546302976,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.7.attn_k.weight",
"offset": 2568323072,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.7.attn_v.weight",
"offset": 2571075584,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.7.attn_output.weight",
"offset": 2576842752,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 2605678592,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.7.ffn_up.weight",
"offset": 2682748928,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.7.ffn_down.weight",
"offset": 2759819264,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.7.attn_norm.weight",
"offset": 2860744704,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 2860777472,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.8.attn_q.weight",
"offset": 2860810240,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.8.attn_k.weight",
"offset": 2882830336,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.8.attn_v.weight",
"offset": 2885582848,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.8.attn_output.weight",
"offset": 2891350016,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 2920185856,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.8.ffn_up.weight",
"offset": 2997256192,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.8.ffn_down.weight",
"offset": 3074326528,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.8.attn_norm.weight",
"offset": 3175251968,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 3175284736,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.9.attn_q.weight",
"offset": 3175317504,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.9.attn_k.weight",
"offset": 3197337600,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.9.attn_v.weight",
"offset": 3200090112,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.9.attn_output.weight",
"offset": 3205857280,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 3234693120,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.9.ffn_up.weight",
"offset": 3311763456,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.9.ffn_down.weight",
"offset": 3388833792,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.9.attn_norm.weight",
"offset": 3489759232,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 3489792000,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.10.attn_q.weight",
"offset": 3489824768,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.10.attn_k.weight",
"offset": 3511844864,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.10.attn_v.weight",
"offset": 3514597376,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.10.attn_output.weight",
"offset": 3520364544,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 3549200384,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.10.ffn_up.weight",
"offset": 3626270720,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.10.ffn_down.weight",
"offset": 3703341056,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.10.attn_norm.weight",
"offset": 3804266496,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 3804299264,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.11.attn_q.weight",
"offset": 3804332032,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.11.attn_k.weight",
"offset": 3826352128,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.11.attn_v.weight",
"offset": 3829104640,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.11.attn_output.weight",
"offset": 3834871808,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 3863707648,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.11.ffn_up.weight",
"offset": 3940777984,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.11.ffn_down.weight",
"offset": 4017848320,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.11.attn_norm.weight",
"offset": 4118773760,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 4118806528,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.12.attn_q.weight",
"offset": 4118839296,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.12.attn_k.weight",
"offset": 4140859392,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.12.attn_v.weight",
"offset": 4143611904,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.12.attn_output.weight",
"offset": 4149379072,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 4178214912,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.12.ffn_up.weight",
"offset": 4255285248,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.12.ffn_down.weight",
"offset": 4332355584,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.12.attn_norm.weight",
"offset": 4433281024,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 4433313792,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.13.attn_q.weight",
"offset": 4433346560,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.13.attn_k.weight",
"offset": 4455366656,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.13.attn_v.weight",
"offset": 4458119168,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.13.attn_output.weight",
"offset": 4463886336,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 4492722176,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.13.ffn_up.weight",
"offset": 4569792512,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.13.ffn_down.weight",
"offset": 4646862848,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.13.attn_norm.weight",
"offset": 4747788288,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 4747821056,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.14.attn_q.weight",
"offset": 4747853824,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.14.attn_k.weight",
"offset": 4769873920,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.14.attn_v.weight",
"offset": 4772626432,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.14.attn_output.weight",
"offset": 4778393600,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 4807229440,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.14.ffn_up.weight",
"offset": 4884299776,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.14.ffn_down.weight",
"offset": 4961370112,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.14.attn_norm.weight",
"offset": 5062295552,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 5062328320,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.15.attn_q.weight",
"offset": 5062361088,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.15.attn_k.weight",
"offset": 5084381184,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.15.attn_v.weight",
"offset": 5087133696,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.15.attn_output.weight",
"offset": 5092900864,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 5121736704,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.15.ffn_up.weight",
"offset": 5198807040,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.15.ffn_down.weight",
"offset": 5275877376,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.15.attn_norm.weight",
"offset": 5376802816,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 5376835584,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.16.attn_q.weight",
"offset": 5376868352,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.16.attn_k.weight",
"offset": 5398888448,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.16.attn_v.weight",
"offset": 5401640960,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.16.attn_output.weight",
"offset": 5407408128,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 5436243968,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.16.ffn_up.weight",
"offset": 5513314304,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.16.ffn_down.weight",
"offset": 5590384640,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.16.attn_norm.weight",
"offset": 5691310080,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 5691342848,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.17.attn_q.weight",
"offset": 5691375616,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.17.attn_k.weight",
"offset": 5713395712,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.17.attn_v.weight",
"offset": 5716148224,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.17.attn_output.weight",
"offset": 5721915392,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 5750751232,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.17.ffn_up.weight",
"offset": 5827821568,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.17.ffn_down.weight",
"offset": 5904891904,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.17.attn_norm.weight",
"offset": 6005817344,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 6005850112,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.18.attn_q.weight",
"offset": 6005882880,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.18.attn_k.weight",
"offset": 6027902976,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.18.attn_v.weight",
"offset": 6030655488,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.18.attn_output.weight",
"offset": 6036422656,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 6065258496,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.18.ffn_up.weight",
"offset": 6142328832,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.18.ffn_down.weight",
"offset": 6219399168,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.18.attn_norm.weight",
"offset": 6320324608,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 6320357376,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.19.attn_q.weight",
"offset": 6320390144,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.19.attn_k.weight",
"offset": 6342410240,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.19.attn_v.weight",
"offset": 6345162752,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.19.attn_output.weight",
"offset": 6350929920,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 6379765760,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.19.ffn_up.weight",
"offset": 6456836096,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.19.ffn_down.weight",
"offset": 6533906432,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.19.attn_norm.weight",
"offset": 6634831872,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 6634864640,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.20.attn_q.weight",
"offset": 6634897408,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.20.attn_k.weight",
"offset": 6656917504,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.20.attn_v.weight",
"offset": 6659670016,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.20.attn_output.weight",
"offset": 6665437184,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 6694273024,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.20.ffn_up.weight",
"offset": 6771343360,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.20.ffn_down.weight",
"offset": 6848413696,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.20.attn_norm.weight",
"offset": 6949339136,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 6949371904,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.21.attn_q.weight",
"offset": 6949404672,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.21.attn_k.weight",
"offset": 6971424768,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.21.attn_v.weight",
"offset": 6974177280,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.21.attn_output.weight",
"offset": 6979944448,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 7008780288,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.21.ffn_up.weight",
"offset": 7085850624,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.21.ffn_down.weight",
"offset": 7162920960,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.21.attn_norm.weight",
"offset": 7263846400,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 7263879168,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.22.attn_q.weight",
"offset": 7263911936,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.22.attn_k.weight",
"offset": 7285932032,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.22.attn_v.weight",
"offset": 7288684544,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.22.attn_output.weight",
"offset": 7294451712,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 7323287552,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.22.ffn_up.weight",
"offset": 7400357888,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.22.ffn_down.weight",
"offset": 7477428224,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.22.attn_norm.weight",
"offset": 7578353664,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 7578386432,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.23.attn_q.weight",
"offset": 7578419200,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.23.attn_k.weight",
"offset": 7600439296,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.23.attn_v.weight",
"offset": 7603191808,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.23.attn_output.weight",
"offset": 7608958976,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 7637794816,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.23.ffn_up.weight",
"offset": 7714865152,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.23.ffn_down.weight",
"offset": 7791935488,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.23.attn_norm.weight",
"offset": 7892860928,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 7892893696,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.24.attn_q.weight",
"offset": 7892926464,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.24.attn_k.weight",
"offset": 7914946560,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.24.attn_v.weight",
"offset": 7917699072,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.24.attn_output.weight",
"offset": 7923466240,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 7952302080,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.24.ffn_up.weight",
"offset": 8029372416,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.24.ffn_down.weight",
"offset": 8106442752,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.24.attn_norm.weight",
"offset": 8207368192,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 8207400960,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.25.attn_q.weight",
"offset": 8207433728,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.25.attn_k.weight",
"offset": 8229453824,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.25.attn_v.weight",
"offset": 8232206336,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.25.attn_output.weight",
"offset": 8237973504,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 8266809344,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.25.ffn_up.weight",
"offset": 8343879680,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.25.ffn_down.weight",
"offset": 8420950016,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.25.attn_norm.weight",
"offset": 8521875456,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 8521908224,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.26.attn_q.weight",
"offset": 8521940992,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.26.attn_k.weight",
"offset": 8543961088,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.26.attn_v.weight",
"offset": 8546713600,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.26.attn_output.weight",
"offset": 8552480768,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 8581316608,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.26.ffn_up.weight",
"offset": 8658386944,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.26.ffn_down.weight",
"offset": 8735457280,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.26.attn_norm.weight",
"offset": 8836382720,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 8836415488,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.27.attn_q.weight",
"offset": 8836448256,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.27.attn_k.weight",
"offset": 8858468352,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.27.attn_v.weight",
"offset": 8861220864,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.27.attn_output.weight",
"offset": 8866988032,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 8895823872,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.27.ffn_up.weight",
"offset": 8972894208,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.27.ffn_down.weight",
"offset": 9049964544,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.27.attn_norm.weight",
"offset": 9150889984,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 9150922752,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.28.attn_q.weight",
"offset": 9150955520,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.28.attn_k.weight",
"offset": 9172975616,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.28.attn_v.weight",
"offset": 9175728128,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.28.attn_output.weight",
"offset": 9181495296,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 9210331136,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.28.ffn_up.weight",
"offset": 9287401472,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.28.ffn_down.weight",
"offset": 9364471808,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.28.attn_norm.weight",
"offset": 9465397248,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 9465430016,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.29.attn_q.weight",
"offset": 9465462784,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.29.attn_k.weight",
"offset": 9487482880,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.29.attn_v.weight",
"offset": 9490235392,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.29.attn_output.weight",
"offset": 9496002560,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 9524838400,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.29.ffn_up.weight",
"offset": 9601908736,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.29.ffn_down.weight",
"offset": 9678979072,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.29.attn_norm.weight",
"offset": 9779904512,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 9779937280,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.30.attn_q.weight",
"offset": 9779970048,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.30.attn_k.weight",
"offset": 9801990144,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.30.attn_v.weight",
"offset": 9804742656,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.30.attn_output.weight",
"offset": 9810509824,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 9839345664,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.30.ffn_up.weight",
"offset": 9916416000,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.30.ffn_down.weight",
"offset": 9993486336,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.30.attn_norm.weight",
"offset": 10094411776,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 10094444544,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.31.attn_q.weight",
"offset": 10094477312,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.31.attn_k.weight",
"offset": 10116497408,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.31.attn_v.weight",
"offset": 10119249920,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.31.attn_output.weight",
"offset": 10125017088,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 10153852928,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.31.ffn_up.weight",
"offset": 10230923264,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.31.ffn_down.weight",
"offset": 10307993600,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.31.attn_norm.weight",
"offset": 10408919040,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 10408951808,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.32.attn_q.weight",
"offset": 10408984576,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.32.attn_k.weight",
"offset": 10431004672,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.32.attn_v.weight",
"offset": 10433757184,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.32.attn_output.weight",
"offset": 10439524352,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 10468360192,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.32.ffn_up.weight",
"offset": 10545430528,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.32.ffn_down.weight",
"offset": 10622500864,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.32.attn_norm.weight",
"offset": 10723426304,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 10723459072,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.33.attn_q.weight",
"offset": 10723491840,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.33.attn_k.weight",
"offset": 10745511936,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.33.attn_v.weight",
"offset": 10748264448,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.33.attn_output.weight",
"offset": 10754031616,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 10782867456,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.33.ffn_up.weight",
"offset": 10859937792,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.33.ffn_down.weight",
"offset": 10937008128,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.33.attn_norm.weight",
"offset": 11037933568,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 11037966336,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.34.attn_q.weight",
"offset": 11037999104,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.34.attn_k.weight",
"offset": 11060019200,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.34.attn_v.weight",
"offset": 11062771712,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.34.attn_output.weight",
"offset": 11068538880,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 11097374720,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.34.ffn_up.weight",
"offset": 11174445056,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.34.ffn_down.weight",
"offset": 11251515392,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.34.attn_norm.weight",
"offset": 11352440832,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 11352473600,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.35.attn_q.weight",
"offset": 11352506368,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.35.attn_k.weight",
"offset": 11374526464,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.35.attn_v.weight",
"offset": 11377278976,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.35.attn_output.weight",
"offset": 11383046144,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 11411881984,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.35.ffn_up.weight",
"offset": 11488952320,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.35.ffn_down.weight",
"offset": 11566022656,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.35.attn_norm.weight",
"offset": 11666948096,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 11666980864,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.36.attn_q.weight",
"offset": 11667013632,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.36.attn_k.weight",
"offset": 11689033728,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.36.attn_v.weight",
"offset": 11691786240,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.36.attn_output.weight",
"offset": 11697553408,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 11726389248,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.36.ffn_up.weight",
"offset": 11803459584,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.36.ffn_down.weight",
"offset": 11880529920,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.36.attn_norm.weight",
"offset": 11981455360,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 11981488128,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.37.attn_q.weight",
"offset": 11981520896,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.37.attn_k.weight",
"offset": 12003540992,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.37.attn_v.weight",
"offset": 12006293504,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.37.attn_output.weight",
"offset": 12012060672,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 12040896512,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.37.ffn_up.weight",
"offset": 12117966848,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.37.ffn_down.weight",
"offset": 12195037184,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.37.attn_norm.weight",
"offset": 12295962624,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 12295995392,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.38.attn_q.weight",
"offset": 12296028160,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.38.attn_k.weight",
"offset": 12318048256,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.38.attn_v.weight",
"offset": 12320800768,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.38.attn_output.weight",
"offset": 12326567936,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 12355403776,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.38.ffn_up.weight",
"offset": 12432474112,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.38.ffn_down.weight",
"offset": 12509544448,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.38.attn_norm.weight",
"offset": 12610469888,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 12610502656,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.39.attn_q.weight",
"offset": 12610535424,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.39.attn_k.weight",
"offset": 12632555520,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.39.attn_v.weight",
"offset": 12635308032,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.39.attn_output.weight",
"offset": 12641075200,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 12669911040,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.39.ffn_up.weight",
"offset": 12746981376,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.39.ffn_down.weight",
"offset": 12824051712,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.39.attn_norm.weight",
"offset": 12924977152,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 12925009920,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.40.attn_q.weight",
"offset": 12925042688,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.40.attn_k.weight",
"offset": 12947062784,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.40.attn_v.weight",
"offset": 12949815296,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.40.attn_output.weight",
"offset": 12955582464,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 12984418304,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.40.ffn_up.weight",
"offset": 13061488640,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.40.ffn_down.weight",
"offset": 13138558976,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.40.attn_norm.weight",
"offset": 13239484416,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 13239517184,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.41.attn_q.weight",
"offset": 13239549952,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.41.attn_k.weight",
"offset": 13261570048,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.41.attn_v.weight",
"offset": 13264322560,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.41.attn_output.weight",
"offset": 13270089728,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 13298925568,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.41.ffn_up.weight",
"offset": 13375995904,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.41.ffn_down.weight",
"offset": 13453066240,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.41.attn_norm.weight",
"offset": 13553991680,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 13554024448,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.42.attn_q.weight",
"offset": 13554057216,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.42.attn_k.weight",
"offset": 13576077312,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.42.attn_v.weight",
"offset": 13578829824,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.42.attn_output.weight",
"offset": 13584596992,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 13613432832,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.42.ffn_up.weight",
"offset": 13690503168,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.42.ffn_down.weight",
"offset": 13767573504,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.42.attn_norm.weight",
"offset": 13868498944,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 13868531712,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.43.attn_q.weight",
"offset": 13868564480,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.43.attn_k.weight",
"offset": 13890584576,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.43.attn_v.weight",
"offset": 13893337088,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.43.attn_output.weight",
"offset": 13899104256,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 13927940096,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.43.ffn_up.weight",
"offset": 14005010432,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.43.ffn_down.weight",
"offset": 14082080768,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.43.attn_norm.weight",
"offset": 14183006208,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 14183038976,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.44.attn_q.weight",
"offset": 14183071744,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.44.attn_k.weight",
"offset": 14205091840,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.44.attn_v.weight",
"offset": 14207844352,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.44.attn_output.weight",
"offset": 14213611520,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 14242447360,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.44.ffn_up.weight",
"offset": 14319517696,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.44.ffn_down.weight",
"offset": 14396588032,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.44.attn_norm.weight",
"offset": 14497513472,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 14497546240,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.45.attn_q.weight",
"offset": 14497579008,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.45.attn_k.weight",
"offset": 14519599104,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.45.attn_v.weight",
"offset": 14522351616,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.45.attn_output.weight",
"offset": 14528118784,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 14556954624,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.45.ffn_up.weight",
"offset": 14634024960,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.45.ffn_down.weight",
"offset": 14711095296,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.45.attn_norm.weight",
"offset": 14812020736,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 14812053504,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.46.attn_q.weight",
"offset": 14812086272,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.46.attn_k.weight",
"offset": 14834106368,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.46.attn_v.weight",
"offset": 14836858880,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.46.attn_output.weight",
"offset": 14842626048,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 14871461888,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.46.ffn_up.weight",
"offset": 14948532224,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.46.ffn_down.weight",
"offset": 15025602560,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.46.attn_norm.weight",
"offset": 15126528000,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 15126560768,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.47.attn_q.weight",
"offset": 15126593536,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.47.attn_k.weight",
"offset": 15148613632,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.47.attn_v.weight",
"offset": 15151366144,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.47.attn_output.weight",
"offset": 15157133312,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 15185969152,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.47.ffn_up.weight",
"offset": 15263039488,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.47.ffn_down.weight",
"offset": 15340109824,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.47.attn_norm.weight",
"offset": 15441035264,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 15441068032,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.48.attn_q.weight",
"offset": 15441100800,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.48.attn_k.weight",
"offset": 15463120896,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.48.attn_v.weight",
"offset": 15465873408,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.48.attn_output.weight",
"offset": 15471640576,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 15500476416,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.48.ffn_up.weight",
"offset": 15577546752,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.48.ffn_down.weight",
"offset": 15654617088,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.48.attn_norm.weight",
"offset": 15755542528,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 15755575296,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.49.attn_q.weight",
"offset": 15755608064,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.49.attn_k.weight",
"offset": 15777628160,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.49.attn_v.weight",
"offset": 15780380672,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.49.attn_output.weight",
"offset": 15786147840,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 15814983680,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.49.ffn_up.weight",
"offset": 15892054016,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.49.ffn_down.weight",
"offset": 15969124352,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.49.attn_norm.weight",
"offset": 16070049792,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 16070082560,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.50.attn_q.weight",
"offset": 16070115328,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.50.attn_k.weight",
"offset": 16092135424,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.50.attn_v.weight",
"offset": 16094887936,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.50.attn_output.weight",
"offset": 16100655104,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 16129490944,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.50.ffn_up.weight",
"offset": 16206561280,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.50.ffn_down.weight",
"offset": 16283631616,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.50.attn_norm.weight",
"offset": 16384557056,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 16384589824,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.51.attn_q.weight",
"offset": 16384622592,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.51.attn_k.weight",
"offset": 16406642688,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.51.attn_v.weight",
"offset": 16409395200,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.51.attn_output.weight",
"offset": 16415162368,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 16443998208,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.51.ffn_up.weight",
"offset": 16521068544,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.51.ffn_down.weight",
"offset": 16598138880,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.51.attn_norm.weight",
"offset": 16699064320,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 16699097088,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.52.attn_q.weight",
"offset": 16699129856,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.52.attn_k.weight",
"offset": 16721149952,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.52.attn_v.weight",
"offset": 16723902464,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.52.attn_output.weight",
"offset": 16729669632,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 16758505472,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.52.ffn_up.weight",
"offset": 16835575808,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.52.ffn_down.weight",
"offset": 16912646144,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.52.attn_norm.weight",
"offset": 17013571584,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 17013604352,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.53.attn_q.weight",
"offset": 17013637120,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.53.attn_k.weight",
"offset": 17035657216,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.53.attn_v.weight",
"offset": 17038409728,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.53.attn_output.weight",
"offset": 17044176896,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 17073012736,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.53.ffn_up.weight",
"offset": 17150083072,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.53.ffn_down.weight",
"offset": 17227153408,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.53.attn_norm.weight",
"offset": 17328078848,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 17328111616,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.54.attn_q.weight",
"offset": 17328144384,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.54.attn_k.weight",
"offset": 17350164480,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.54.attn_v.weight",
"offset": 17352916992,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.54.attn_output.weight",
"offset": 17358684160,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 17387520000,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.54.ffn_up.weight",
"offset": 17464590336,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.54.ffn_down.weight",
"offset": 17541660672,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.54.attn_norm.weight",
"offset": 17642586112,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 17642618880,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.55.attn_q.weight",
"offset": 17642651648,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.55.attn_k.weight",
"offset": 17664671744,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.55.attn_v.weight",
"offset": 17667424256,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.55.attn_output.weight",
"offset": 17673191424,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 17702027264,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.55.ffn_up.weight",
"offset": 17779097600,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.55.ffn_down.weight",
"offset": 17856167936,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.55.attn_norm.weight",
"offset": 17957093376,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 17957126144,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.56.attn_q.weight",
"offset": 17957158912,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.56.attn_k.weight",
"offset": 17979179008,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.56.attn_v.weight",
"offset": 17981931520,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.56.attn_output.weight",
"offset": 17987698688,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 18016534528,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.56.ffn_up.weight",
"offset": 18093604864,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.56.ffn_down.weight",
"offset": 18170675200,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.56.attn_norm.weight",
"offset": 18271600640,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 18271633408,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.57.attn_q.weight",
"offset": 18271666176,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.57.attn_k.weight",
"offset": 18293686272,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.57.attn_v.weight",
"offset": 18296438784,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.57.attn_output.weight",
"offset": 18302205952,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 18331041792,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.57.ffn_up.weight",
"offset": 18408112128,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.57.ffn_down.weight",
"offset": 18485182464,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.57.attn_norm.weight",
"offset": 18586107904,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 18586140672,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.58.attn_q.weight",
"offset": 18586173440,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.58.attn_k.weight",
"offset": 18608193536,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.58.attn_v.weight",
"offset": 18610946048,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.58.attn_output.weight",
"offset": 18616713216,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 18645549056,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.58.ffn_up.weight",
"offset": 18722619392,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.58.ffn_down.weight",
"offset": 18799689728,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.58.attn_norm.weight",
"offset": 18900615168,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 18900647936,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.59.attn_q.weight",
"offset": 18900680704,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.59.attn_k.weight",
"offset": 18922700800,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.59.attn_v.weight",
"offset": 18925453312,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.59.attn_output.weight",
"offset": 18931220480,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 18960056320,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.59.ffn_up.weight",
"offset": 19037126656,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.59.ffn_down.weight",
"offset": 19114196992,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.59.attn_norm.weight",
"offset": 19215122432,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 19215155200,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.60.attn_q.weight",
"offset": 19215187968,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "\n\u0000\u0000\u0000\u0000\ufffd\ufffdz\u0004\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0000\u0000\u0000\u0000",
"offset": 19237208064,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.60.attn_v.weight",
"offset": 19239960576,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.60.attn_output.weight",
"offset": 19245727744,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 19274563584,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.60.ffn_up.weight",
"offset": 19351633920,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.60.ffn_down.weight",
"offset": 19428704256,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.60.attn_norm.weight",
"offset": 19529629696,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 19529662464,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.61.attn_q.weight",
"offset": 19529695232,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.61.attn_k.weight",
"offset": 19551715328,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.61.attn_v.weight",
"offset": 19554467840,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.61.attn_output.weight",
"offset": 19560235008,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 19589070848,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.61.ffn_up.weight",
"offset": 19666141184,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.61.ffn_down.weight",
"offset": 19743211520,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.61.attn_norm.weight",
"offset": 19844136960,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 19844169728,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.62.attn_q.weight",
"offset": 19844202496,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.62.attn_k.weight",
"offset": 19866222592,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.62.attn_v.weight",
"offset": 19868975104,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.62.attn_output.weight",
"offset": 19874742272,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 19903578112,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.62.ffn_up.weight",
"offset": 19980648448,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.62.ffn_down.weight",
"offset": 20057718784,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.62.attn_norm.weight",
"offset": 20158644224,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 20158676992,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.63.attn_q.weight",
"offset": 20158709760,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.63.attn_k.weight",
"offset": 20180729856,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.63.attn_v.weight",
"offset": 20183482368,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.63.attn_output.weight",
"offset": 20189249536,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 20218085376,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.63.ffn_up.weight",
"offset": 20295155712,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.63.ffn_down.weight",
"offset": 20372226048,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.63.attn_norm.weight",
"offset": 20473151488,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 20473184256,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.64.attn_q.weight",
"offset": 20473217024,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.64.attn_k.weight",
"offset": 20495237120,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.64.attn_v.weight",
"offset": 20497989632,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.64.attn_output.weight",
"offset": 20503756800,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 20532592640,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.64.ffn_up.weight",
"offset": 20609662976,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.64.ffn_down.weight",
"offset": 20686733312,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.64.attn_norm.weight",
"offset": 20787658752,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 20787691520,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.65.attn_q.weight",
"offset": 20787724288,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.65.attn_k.weight",
"offset": 20809744384,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.65.attn_v.weight",
"offset": 20812496896,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.65.attn_output.weight",
"offset": 20818264064,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 20847099904,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.65.ffn_up.weight",
"offset": 20924170240,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.65.ffn_down.weight",
"offset": 21001240576,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.65.attn_norm.weight",
"offset": 21102166016,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 21102198784,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.66.attn_q.weight",
"offset": 21102231552,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.66.attn_k.weight",
"offset": 21124251648,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.66.attn_v.weight",
"offset": 21127004160,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.66.attn_output.weight",
"offset": 21132771328,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 21161607168,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.66.ffn_up.weight",
"offset": 21238677504,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.66.ffn_down.weight",
"offset": 21315747840,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.66.attn_norm.weight",
"offset": 21416673280,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 21416706048,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.67.attn_q.weight",
"offset": 21416738816,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.67.attn_k.weight",
"offset": 21438758912,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.67.attn_v.weight",
"offset": 21441511424,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.67.attn_output.weight",
"offset": 21447278592,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 21476114432,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.67.ffn_up.weight",
"offset": 21553184768,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.67.ffn_down.weight",
"offset": 21630255104,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.67.attn_norm.weight",
"offset": 21731180544,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 21731213312,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.68.attn_q.weight",
"offset": 21731246080,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.68.attn_k.weight",
"offset": 21753266176,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.68.attn_v.weight",
"offset": 21756018688,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.68.attn_output.weight",
"offset": 21761785856,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 21790621696,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.68.ffn_up.weight",
"offset": 21867692032,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.68.ffn_down.weight",
"offset": 21944762368,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.68.attn_norm.weight",
"offset": 22045687808,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 22045720576,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.69.attn_q.weight",
"offset": 22045753344,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.69.attn_k.weight",
"offset": 22067773440,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.69.attn_v.weight",
"offset": 22070525952,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.69.attn_output.weight",
"offset": 22076293120,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 22105128960,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.69.ffn_up.weight",
"offset": 22182199296,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.69.ffn_down.weight",
"offset": 22259269632,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.69.attn_norm.weight",
"offset": 22360195072,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 22360227840,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.70.attn_q.weight",
"offset": 22360260608,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.70.attn_k.weight",
"offset": 22382280704,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.70.attn_v.weight",
"offset": 22385033216,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.70.attn_output.weight",
"offset": 22390800384,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 22419636224,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.70.ffn_up.weight",
"offset": 22496706560,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.70.ffn_down.weight",
"offset": 22573776896,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.70.attn_norm.weight",
"offset": 22674702336,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 22674735104,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.71.attn_q.weight",
"offset": 22674767872,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.71.attn_k.weight",
"offset": 22696787968,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.71.attn_v.weight",
"offset": 22699540480,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.71.attn_output.weight",
"offset": 22705307648,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 22734143488,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.71.ffn_up.weight",
"offset": 22811213824,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.71.ffn_down.weight",
"offset": 22888284160,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.71.attn_norm.weight",
"offset": 22989209600,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 22989242368,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.72.attn_q.weight",
"offset": 22989275136,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.72.attn_k.weight",
"offset": 23011295232,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.72.attn_v.weight",
"offset": 23014047744,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.72.attn_output.weight",
"offset": 23019814912,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 23048650752,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.72.ffn_up.weight",
"offset": 23125721088,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.72.ffn_down.weight",
"offset": 23202791424,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.72.attn_norm.weight",
"offset": 23303716864,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 23303749632,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.73.attn_q.weight",
"offset": 23303782400,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.73.attn_k.weight",
"offset": 23325802496,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.73.attn_v.weight",
"offset": 23328555008,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.73.attn_output.weight",
"offset": 23334322176,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 23363158016,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.73.ffn_up.weight",
"offset": 23440228352,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.73.ffn_down.weight",
"offset": 23517298688,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.73.attn_norm.weight",
"offset": 23618224128,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 23618256896,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.74.attn_q.weight",
"offset": 23618289664,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.74.attn_k.weight",
"offset": 23640309760,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.74.attn_v.weight",
"offset": 23643062272,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.74.attn_output.weight",
"offset": 23648829440,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 23677665280,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.74.ffn_up.weight",
"offset": 23754735616,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.74.ffn_down.weight",
"offset": 23831805952,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.74.attn_norm.weight",
"offset": 23932731392,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 23932764160,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.75.attn_q.weight",
"offset": 23932796928,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.75.attn_k.weight",
"offset": 23954817024,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.75.attn_v.weight",
"offset": 23957569536,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "w7|\u001a{\ufffdIU\u0004UD\u00139h\u0015\u001a\u0014\u0005\u0004\ufffdq\u001cYU#",
"offset": 23963336704,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 23992172544,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.75.ffn_up.weight",
"offset": 24069242880,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.75.ffn_down.weight",
"offset": 24146313216,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.75.attn_norm.weight",
"offset": 24247238656,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 24247271424,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.76.attn_q.weight",
"offset": 24247304192,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.76.attn_k.weight",
"offset": 24269324288,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.76.attn_v.weight",
"offset": 24272076800,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.76.attn_output.weight",
"offset": 24277843968,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 24306679808,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.76.ffn_up.weight",
"offset": 24383750144,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.76.ffn_down.weight",
"offset": 24460820480,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.76.attn_norm.weight",
"offset": 24561745920,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 24561778688,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.77.attn_q.weight",
"offset": 24561811456,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.77.attn_k.weight",
"offset": 24583831552,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.77.attn_v.weight",
"offset": 24586584064,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.77.attn_output.weight",
"offset": 24592351232,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 24621187072,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.77.ffn_up.weight",
"offset": 24698257408,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.77.ffn_down.weight",
"offset": 24775327744,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.77.attn_norm.weight",
"offset": 24876253184,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 24876285952,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.78.attn_q.weight",
"offset": 24876318720,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.78.attn_k.weight",
"offset": 24898338816,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.78.attn_v.weight",
"offset": 24901091328,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.78.attn_output.weight",
"offset": 24906858496,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 24935694336,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.78.ffn_up.weight",
"offset": 25012764672,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.78.ffn_down.weight",
"offset": 25089835008,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.78.attn_norm.weight",
"offset": 25190760448,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 25190793216,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "blk.79.attn_q.weight",
"offset": 25190825984,
"shape": [
8192,
8192
],
"size": 32768,
"type": 10
},
{
"name": "blk.79.attn_k.weight",
"offset": 25212846080,
"shape": [
8192,
1024
],
"size": 22020096,
"type": 10
},
{
"name": "blk.79.attn_v.weight",
"offset": 25215598592,
"shape": [
8192,
1024
],
"size": 2752512,
"type": 13
},
{
"name": "blk.79.attn_output.weight",
"offset": 25221365760,
"shape": [
8192,
8192
],
"size": 5767168,
"type": 11
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 25250201600,
"shape": [
8192,
28672
],
"size": 28835840,
"type": 10
},
{
"name": "blk.79.ffn_up.weight",
"offset": 25327271936,
"shape": [
8192,
28672
],
"size": 77070336,
"type": 10
},
{
"name": "blk.79.ffn_down.weight",
"offset": 25404342272,
"shape": [
28672,
8192
],
"size": 77070336,
"type": 11
},
{
"name": "blk.79.attn_norm.weight",
"offset": 25505267712,
"shape": [
8192
],
"size": 100925440,
"type": 0
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 25505300480,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output_norm.weight",
"offset": 25505333248,
"shape": [
8192
],
"size": 32768,
"type": 0
},
{
"name": "output.weight",
"offset": 25505366016,
"shape": [
8192,
128256
],
"size": 32768,
"type": 14
}
],
"version": 3
}