This model was created with the goal for a good llama 3 uncencored model with long context. At it worked like a charm.
70B
1,059 Pulls Updated 5 weeks ago
f3e08b1017a8 · 43GB
{
"metadata": {
"general.architecture": "llama",
"general.file_type": 15,
"general.name": "Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.quantization_version": 2,
"general.source.url": "https://huggingface.co/ryzen88/Llama-3-70b-Uncensored-Lumi-Tess-gradient",
"general.url": "https://huggingface.co/mradermacher/Llama-3-70b-Uncensored-Lumi-Tess-gradient-i1-GGUF",
"llama.attention.head_count": 64,
"llama.attention.head_count_kv": 8,
"llama.attention.layer_norm_rms_epsilon": 0.00001,
"llama.block_count": 80,
"llama.context_length": 262144,
"llama.embedding_length": 8192,
"llama.feed_forward_length": 28672,
"llama.rope.dimension_count": 128,
"llama.rope.freq_base": 207112200,
"llama.vocab_size": 128256,
"mradermacher.convert_type": "hf",
"mradermacher.quantize_version": "2",
"mradermacher.quantized_at": "2024-05-15T10:57:16+02:00",
"mradermacher.quantized_by": "mradermacher",
"mradermacher.quantized_on": "db1",
"quantize.imatrix.chunks_count": 277,
"quantize.imatrix.dataset": "imatrix-training-full.txt",
"quantize.imatrix.entries_count": 560,
"quantize.imatrix.file": "Llama-3-70b-Uncensored-Lumi-Tess-gradient-i1-GGUF/imatrix.dat",
"tokenizer.ggml.bos_token_id": 128000,
"tokenizer.ggml.eos_token_id": 128001,
"tokenizer.ggml.merges": "... (280147 values)",
"tokenizer.ggml.model": "gpt2",
"tokenizer.ggml.pre": "llama-bpe",
"tokenizer.ggml.token_type": "... (128256 values)",
"tokenizer.ggml.tokens": "... (128256 values)"
},
"num_params": 70553706496,
"tensors": [
{
"name": "token_embd.weight",
"offset": 861880320,
"shape": [
8192,
128256
],
"size": 861880320,
"type": 12
},
{
"name": "blk.0.attn_norm.weight",
"offset": 1452883968,
"shape": [
8192
],
"size": 591003648,
"type": 0
},
{
"name": "blk.0.ffn_down.weight",
"offset": 1452916736,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.0.ffn_gate.weight",
"offset": 1645592576,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.0.ffn_up.weight",
"offset": 1777713152,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.0.ffn_norm.weight",
"offset": 1909833728,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.0.attn_k.weight",
"offset": 1909866496,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.0.attn_output.weight",
"offset": 1914585088,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.0.attn_q.weight",
"offset": 1952333824,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.0.attn_v.weight",
"offset": 1990082560,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.1.attn_norm.weight",
"offset": 1996963840,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.1.ffn_down.weight",
"offset": 1996996608,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.1.ffn_gate.weight",
"offset": 2189672448,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.1.ffn_up.weight",
"offset": 2321793024,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.1.ffn_norm.weight",
"offset": 2453913600,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.1.attn_k.weight",
"offset": 2453946368,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.1.attn_output.weight",
"offset": 2458664960,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.1.attn_q.weight",
"offset": 2496413696,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.1.attn_v.weight",
"offset": 2534162432,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.10.attn_norm.weight",
"offset": 2541043712,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.10.ffn_down.weight",
"offset": 2541076480,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.10.ffn_gate.weight",
"offset": 2733752320,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.10.ffn_up.weight",
"offset": 2865872896,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.10.ffn_norm.weight",
"offset": 2997993472,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.10.attn_k.weight",
"offset": 2998026240,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.10.attn_output.weight",
"offset": 3002744832,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.10.attn_q.weight",
"offset": 3040493568,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.10.attn_v.weight",
"offset": 3078242304,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.11.attn_norm.weight",
"offset": 3085123584,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.11.ffn_down.weight",
"offset": 3085156352,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.11.ffn_gate.weight",
"offset": 3277832192,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.11.ffn_up.weight",
"offset": 3409952768,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.11.ffn_norm.weight",
"offset": 3542073344,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.11.attn_k.weight",
"offset": 3542106112,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.11.attn_output.weight",
"offset": 3546824704,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.11.attn_q.weight",
"offset": 3584573440,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.11.attn_v.weight",
"offset": 3622322176,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.12.attn_norm.weight",
"offset": 3629203456,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.12.ffn_down.weight",
"offset": 3629236224,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.12.ffn_gate.weight",
"offset": 3821912064,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.12.ffn_up.weight",
"offset": 3954032640,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.12.ffn_norm.weight",
"offset": 4086153216,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.12.attn_k.weight",
"offset": 4086185984,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.12.attn_output.weight",
"offset": 4090904576,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.12.attn_q.weight",
"offset": 4128653312,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.12.attn_v.weight",
"offset": 4166402048,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.13.attn_norm.weight",
"offset": 4173283328,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.13.ffn_down.weight",
"offset": 4173316096,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.13.ffn_gate.weight",
"offset": 4365991936,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.13.ffn_up.weight",
"offset": 4498112512,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.13.ffn_norm.weight",
"offset": 4630233088,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.13.attn_k.weight",
"offset": 4630265856,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.13.attn_output.weight",
"offset": 4634984448,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.13.attn_q.weight",
"offset": 4672733184,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.13.attn_v.weight",
"offset": 4710481920,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.14.attn_norm.weight",
"offset": 4717363200,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.14.ffn_down.weight",
"offset": 4717395968,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.14.ffn_gate.weight",
"offset": 4910071808,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.14.ffn_up.weight",
"offset": 5042192384,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.14.ffn_norm.weight",
"offset": 5174312960,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.14.attn_k.weight",
"offset": 5174345728,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.14.attn_output.weight",
"offset": 5179064320,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.14.attn_q.weight",
"offset": 5216813056,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.14.attn_v.weight",
"offset": 5254561792,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.15.attn_norm.weight",
"offset": 5261443072,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.15.ffn_down.weight",
"offset": 5261475840,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.15.ffn_gate.weight",
"offset": 5454151680,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.15.ffn_up.weight",
"offset": 5586272256,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.15.ffn_norm.weight",
"offset": 5718392832,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.15.attn_k.weight",
"offset": 5718425600,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.15.attn_output.weight",
"offset": 5723144192,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.15.attn_q.weight",
"offset": 5760892928,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.15.attn_v.weight",
"offset": 5798641664,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.16.attn_norm.weight",
"offset": 5805522944,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.16.ffn_down.weight",
"offset": 5805555712,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.16.ffn_gate.weight",
"offset": 5998231552,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.16.ffn_up.weight",
"offset": 6130352128,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.16.ffn_norm.weight",
"offset": 6262472704,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.16.attn_k.weight",
"offset": 6262505472,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.16.attn_output.weight",
"offset": 6267224064,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.16.attn_q.weight",
"offset": 6304972800,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.16.attn_v.weight",
"offset": 6342721536,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.17.attn_norm.weight",
"offset": 6349602816,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.17.ffn_down.weight",
"offset": 6349635584,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.17.ffn_gate.weight",
"offset": 6542311424,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.17.ffn_up.weight",
"offset": 6674432000,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.17.ffn_norm.weight",
"offset": 6806552576,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.17.attn_k.weight",
"offset": 6806585344,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.17.attn_output.weight",
"offset": 6811303936,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.17.attn_q.weight",
"offset": 6849052672,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.17.attn_v.weight",
"offset": 6886801408,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.18.attn_norm.weight",
"offset": 6893682688,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.18.ffn_down.weight",
"offset": 6893715456,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.18.ffn_gate.weight",
"offset": 7025836032,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.18.ffn_up.weight",
"offset": 7157956608,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.18.ffn_norm.weight",
"offset": 7290077184,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.18.attn_k.weight",
"offset": 7290109952,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.18.attn_output.weight",
"offset": 7294828544,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.18.attn_q.weight",
"offset": 7332577280,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.18.attn_v.weight",
"offset": 7370326016,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.19.attn_norm.weight",
"offset": 7376093184,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.19.ffn_down.weight",
"offset": 7376125952,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.19.ffn_gate.weight",
"offset": 7508246528,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.19.ffn_up.weight",
"offset": 7640367104,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.19.ffn_norm.weight",
"offset": 7772487680,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.19.attn_k.weight",
"offset": 7772520448,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.19.attn_output.weight",
"offset": 7777239040,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.19.attn_q.weight",
"offset": 7814987776,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.19.attn_v.weight",
"offset": 7852736512,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.2.attn_norm.weight",
"offset": 7858503680,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.2.ffn_down.weight",
"offset": 7858536448,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.2.ffn_gate.weight",
"offset": 8051212288,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.2.ffn_up.weight",
"offset": 8183332864,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.2.ffn_norm.weight",
"offset": 8315453440,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.2.attn_k.weight",
"offset": 8315486208,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.2.attn_output.weight",
"offset": 8320204800,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.2.attn_q.weight",
"offset": 8357953536,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.2.attn_v.weight",
"offset": 8395702272,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.20.attn_norm.weight",
"offset": 8402583552,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.20.ffn_down.weight",
"offset": 8402616320,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.20.ffn_gate.weight",
"offset": 8534736896,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.20.ffn_up.weight",
"offset": 8666857472,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.20.ffn_norm.weight",
"offset": 8798978048,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.20.attn_k.weight",
"offset": 8799010816,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.20.attn_output.weight",
"offset": 8803729408,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.20.attn_q.weight",
"offset": 8841478144,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.20.attn_v.weight",
"offset": 8879226880,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.21.attn_norm.weight",
"offset": 8884994048,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.21.ffn_down.weight",
"offset": 8885026816,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.21.ffn_gate.weight",
"offset": 9017147392,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.21.ffn_up.weight",
"offset": 9149267968,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.21.ffn_norm.weight",
"offset": 9281388544,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.21.attn_k.weight",
"offset": 9281421312,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.21.attn_output.weight",
"offset": 9286139904,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.21.attn_q.weight",
"offset": 9323888640,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.21.attn_v.weight",
"offset": 9361637376,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.22.attn_norm.weight",
"offset": 9367404544,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.22.ffn_down.weight",
"offset": 9367437312,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.22.ffn_gate.weight",
"offset": 9560113152,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.22.ffn_up.weight",
"offset": 9692233728,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.22.ffn_norm.weight",
"offset": 9824354304,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.22.attn_k.weight",
"offset": 9824387072,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.22.attn_output.weight",
"offset": 9829105664,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.22.attn_q.weight",
"offset": 9866854400,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.22.attn_v.weight",
"offset": 9904603136,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.23.attn_norm.weight",
"offset": 9911484416,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.23.ffn_down.weight",
"offset": 9911517184,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.23.ffn_gate.weight",
"offset": 10043637760,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.23.ffn_up.weight",
"offset": 10175758336,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.23.ffn_norm.weight",
"offset": 10307878912,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.23.attn_k.weight",
"offset": 10307911680,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.23.attn_output.weight",
"offset": 10312630272,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.23.attn_q.weight",
"offset": 10350379008,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.23.attn_v.weight",
"offset": 10388127744,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.24.attn_norm.weight",
"offset": 10393894912,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.24.ffn_down.weight",
"offset": 10393927680,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.24.ffn_gate.weight",
"offset": 10526048256,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.24.ffn_up.weight",
"offset": 10658168832,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.24.ffn_norm.weight",
"offset": 10790289408,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.24.attn_k.weight",
"offset": 10790322176,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.24.attn_output.weight",
"offset": 10795040768,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.24.attn_q.weight",
"offset": 10832789504,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.24.attn_v.weight",
"offset": 10870538240,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.25.attn_norm.weight",
"offset": 10876305408,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.25.ffn_down.weight",
"offset": 10876338176,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.25.ffn_gate.weight",
"offset": 11069014016,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.25.ffn_up.weight",
"offset": 11201134592,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.25.ffn_norm.weight",
"offset": 11333255168,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.25.attn_k.weight",
"offset": 11333287936,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.25.attn_output.weight",
"offset": 11338006528,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.25.attn_q.weight",
"offset": 11375755264,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.25.attn_v.weight",
"offset": 11413504000,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.26.attn_norm.weight",
"offset": 11420385280,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.26.ffn_down.weight",
"offset": 11420418048,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.26.ffn_gate.weight",
"offset": 11552538624,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.26.ffn_up.weight",
"offset": 11684659200,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.26.ffn_norm.weight",
"offset": 11816779776,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.26.attn_k.weight",
"offset": 11816812544,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.26.attn_output.weight",
"offset": 11821531136,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.26.attn_q.weight",
"offset": 11859279872,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.26.attn_v.weight",
"offset": 11897028608,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.27.attn_norm.weight",
"offset": 11902795776,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.27.ffn_down.weight",
"offset": 11902828544,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.27.ffn_gate.weight",
"offset": 12034949120,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.27.ffn_up.weight",
"offset": 12167069696,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.27.ffn_norm.weight",
"offset": 12299190272,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.27.attn_k.weight",
"offset": 12299223040,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.27.attn_output.weight",
"offset": 12303941632,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.27.attn_q.weight",
"offset": 12341690368,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.27.attn_v.weight",
"offset": 12379439104,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.28.attn_norm.weight",
"offset": 12385206272,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.28.ffn_down.weight",
"offset": 12385239040,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.28.ffn_gate.weight",
"offset": 12577914880,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.28.ffn_up.weight",
"offset": 12710035456,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.28.ffn_norm.weight",
"offset": 12842156032,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.28.attn_k.weight",
"offset": 12842188800,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.28.attn_output.weight",
"offset": 12846907392,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.28.attn_q.weight",
"offset": 12884656128,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.28.attn_v.weight",
"offset": 12922404864,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.29.attn_norm.weight",
"offset": 12929286144,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.29.ffn_down.weight",
"offset": 12929318912,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.29.ffn_gate.weight",
"offset": 13061439488,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.29.ffn_up.weight",
"offset": 13193560064,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.29.ffn_norm.weight",
"offset": 13325680640,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.29.attn_k.weight",
"offset": 13325713408,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.29.attn_output.weight",
"offset": 13330432000,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.29.attn_q.weight",
"offset": 13368180736,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.29.attn_v.weight",
"offset": 13405929472,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.3.attn_norm.weight",
"offset": 13411696640,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.3.ffn_down.weight",
"offset": 13411729408,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.3.ffn_gate.weight",
"offset": 13543849984,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.3.ffn_up.weight",
"offset": 13675970560,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.3.ffn_norm.weight",
"offset": 13808091136,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.3.attn_k.weight",
"offset": 13808123904,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.3.attn_output.weight",
"offset": 13812842496,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.3.attn_q.weight",
"offset": 13850591232,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.3.attn_v.weight",
"offset": 13888339968,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.30.attn_norm.weight",
"offset": 13894107136,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.30.ffn_down.weight",
"offset": 13894139904,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.30.ffn_gate.weight",
"offset": 14086815744,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.30.ffn_up.weight",
"offset": 14218936320,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.30.ffn_norm.weight",
"offset": 14351056896,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.30.attn_k.weight",
"offset": 14351089664,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.30.attn_output.weight",
"offset": 14355808256,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.30.attn_q.weight",
"offset": 14393556992,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.30.attn_v.weight",
"offset": 14431305728,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.31.attn_norm.weight",
"offset": 14438187008,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.31.ffn_down.weight",
"offset": 14438219776,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.31.ffn_gate.weight",
"offset": 14570340352,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.31.ffn_up.weight",
"offset": 14702460928,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.31.ffn_norm.weight",
"offset": 14834581504,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.31.attn_k.weight",
"offset": 14834614272,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.31.attn_output.weight",
"offset": 14839332864,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.31.attn_q.weight",
"offset": 14877081600,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.31.attn_v.weight",
"offset": 14914830336,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.32.attn_norm.weight",
"offset": 14920597504,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.32.ffn_down.weight",
"offset": 14920630272,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.32.ffn_gate.weight",
"offset": 15052750848,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.32.ffn_up.weight",
"offset": 15184871424,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.32.ffn_norm.weight",
"offset": 15316992000,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.32.attn_k.weight",
"offset": 15317024768,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.32.attn_output.weight",
"offset": 15321743360,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.32.attn_q.weight",
"offset": 15359492096,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.32.attn_v.weight",
"offset": 15397240832,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.33.attn_norm.weight",
"offset": 15403008000,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.33.ffn_down.weight",
"offset": 15403040768,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.33.ffn_gate.weight",
"offset": 15595716608,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.33.ffn_up.weight",
"offset": 15727837184,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.33.ffn_norm.weight",
"offset": 15859957760,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.33.attn_k.weight",
"offset": 15859990528,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.33.attn_output.weight",
"offset": 15864709120,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.33.attn_q.weight",
"offset": 15902457856,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.33.attn_v.weight",
"offset": 15940206592,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.34.attn_norm.weight",
"offset": 15947087872,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.34.ffn_down.weight",
"offset": 15947120640,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.34.ffn_gate.weight",
"offset": 16079241216,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.34.ffn_up.weight",
"offset": 16211361792,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.34.ffn_norm.weight",
"offset": 16343482368,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.34.attn_k.weight",
"offset": 16343515136,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.34.attn_output.weight",
"offset": 16348233728,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.34.attn_q.weight",
"offset": 16385982464,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.34.attn_v.weight",
"offset": 16423731200,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.35.attn_norm.weight",
"offset": 16429498368,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.35.ffn_down.weight",
"offset": 16429531136,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.35.ffn_gate.weight",
"offset": 16561651712,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.35.ffn_up.weight",
"offset": 16693772288,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.35.ffn_norm.weight",
"offset": 16825892864,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.35.attn_k.weight",
"offset": 16825925632,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.35.attn_output.weight",
"offset": 16830644224,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.35.attn_q.weight",
"offset": 16868392960,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.35.attn_v.weight",
"offset": 16906141696,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.36.attn_norm.weight",
"offset": 16911908864,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.36.ffn_down.weight",
"offset": 16911941632,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.36.ffn_gate.weight",
"offset": 17104617472,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.36.ffn_up.weight",
"offset": 17236738048,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.36.ffn_norm.weight",
"offset": 17368858624,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.36.attn_k.weight",
"offset": 17368891392,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.36.attn_output.weight",
"offset": 17373609984,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.36.attn_q.weight",
"offset": 17411358720,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.36.attn_v.weight",
"offset": 17449107456,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.37.attn_norm.weight",
"offset": 17455988736,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.37.ffn_down.weight",
"offset": 17456021504,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.37.ffn_gate.weight",
"offset": 17588142080,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.37.ffn_up.weight",
"offset": 17720262656,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.37.ffn_norm.weight",
"offset": 17852383232,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.37.attn_k.weight",
"offset": 17852416000,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.37.attn_output.weight",
"offset": 17857134592,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.37.attn_q.weight",
"offset": 17894883328,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.37.attn_v.weight",
"offset": 17932632064,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.38.attn_norm.weight",
"offset": 17938399232,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.38.ffn_down.weight",
"offset": 17938432000,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.38.ffn_gate.weight",
"offset": 18070552576,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.38.ffn_up.weight",
"offset": 18202673152,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.38.ffn_norm.weight",
"offset": 18334793728,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.38.attn_k.weight",
"offset": 18334826496,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.38.attn_output.weight",
"offset": 18339545088,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.38.attn_q.weight",
"offset": 18377293824,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.38.attn_v.weight",
"offset": 18415042560,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.39.attn_norm.weight",
"offset": 18420809728,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.39.ffn_down.weight",
"offset": 18420842496,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.39.ffn_gate.weight",
"offset": 18613518336,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.39.ffn_up.weight",
"offset": 18745638912,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.39.ffn_norm.weight",
"offset": 18877759488,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.39.attn_k.weight",
"offset": 18877792256,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.39.attn_output.weight",
"offset": 18882510848,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.39.attn_q.weight",
"offset": 18920259584,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.39.attn_v.weight",
"offset": 18958008320,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.4.attn_norm.weight",
"offset": 18964889600,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.4.ffn_down.weight",
"offset": 18964922368,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.4.ffn_gate.weight",
"offset": 19097042944,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.4.ffn_up.weight",
"offset": 19229163520,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.4.ffn_norm.weight",
"offset": 19361284096,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.4.attn_k.weight",
"offset": 19361316864,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.4.attn_output.weight",
"offset": 19366035456,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.4.attn_q.weight",
"offset": 19403784192,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.4.attn_v.weight",
"offset": 19441532928,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.40.attn_norm.weight",
"offset": 19447300096,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.40.ffn_down.weight",
"offset": 19447332864,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.40.ffn_gate.weight",
"offset": 19579453440,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.40.ffn_up.weight",
"offset": 19711574016,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.40.ffn_norm.weight",
"offset": 19843694592,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.40.attn_k.weight",
"offset": 19843727360,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.40.attn_output.weight",
"offset": 19848445952,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.40.attn_q.weight",
"offset": 19886194688,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.40.attn_v.weight",
"offset": 19923943424,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.41.attn_norm.weight",
"offset": 19929710592,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.41.ffn_down.weight",
"offset": 19929743360,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.41.ffn_gate.weight",
"offset": 20122419200,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.41.ffn_up.weight",
"offset": 20254539776,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.41.ffn_norm.weight",
"offset": 20386660352,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.41.attn_k.weight",
"offset": 20386693120,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.41.attn_output.weight",
"offset": 20391411712,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.41.attn_q.weight",
"offset": 20429160448,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.41.attn_v.weight",
"offset": 20466909184,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.42.attn_norm.weight",
"offset": 20473790464,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.42.ffn_down.weight",
"offset": 20473823232,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.42.ffn_gate.weight",
"offset": 20605943808,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.42.ffn_up.weight",
"offset": 20738064384,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.42.ffn_norm.weight",
"offset": 20870184960,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.42.attn_k.weight",
"offset": 20870217728,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.42.attn_output.weight",
"offset": 20874936320,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.42.attn_q.weight",
"offset": 20912685056,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.42.attn_v.weight",
"offset": 20950433792,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.43.attn_norm.weight",
"offset": 20956200960,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.43.ffn_down.weight",
"offset": 20956233728,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.43.ffn_gate.weight",
"offset": 21088354304,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.43.ffn_up.weight",
"offset": 21220474880,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.43.ffn_norm.weight",
"offset": 21352595456,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.43.attn_k.weight",
"offset": 21352628224,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.43.attn_output.weight",
"offset": 21357346816,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.43.attn_q.weight",
"offset": 21395095552,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.43.attn_v.weight",
"offset": 21432844288,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.44.attn_norm.weight",
"offset": 21438611456,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.44.ffn_down.weight",
"offset": 21438644224,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.44.ffn_gate.weight",
"offset": 21631320064,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.44.ffn_up.weight",
"offset": 21763440640,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.44.ffn_norm.weight",
"offset": 21895561216,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.44.attn_k.weight",
"offset": 21895593984,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.44.attn_output.weight",
"offset": 21900312576,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.44.attn_q.weight",
"offset": 21938061312,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.44.attn_v.weight",
"offset": 21975810048,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.45.attn_norm.weight",
"offset": 21982691328,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.45.ffn_down.weight",
"offset": 21982724096,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.45.ffn_gate.weight",
"offset": 22114844672,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.45.ffn_up.weight",
"offset": 22246965248,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.45.ffn_norm.weight",
"offset": 22379085824,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.45.attn_k.weight",
"offset": 22379118592,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.45.attn_output.weight",
"offset": 22383837184,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.45.attn_q.weight",
"offset": 22421585920,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.45.attn_v.weight",
"offset": 22459334656,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.46.attn_norm.weight",
"offset": 22465101824,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.46.ffn_down.weight",
"offset": 22465134592,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.46.ffn_gate.weight",
"offset": 22597255168,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.46.ffn_up.weight",
"offset": 22729375744,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.46.ffn_norm.weight",
"offset": 22861496320,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.46.attn_k.weight",
"offset": 22861529088,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.46.attn_output.weight",
"offset": 22866247680,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.46.attn_q.weight",
"offset": 22903996416,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.46.attn_v.weight",
"offset": 22941745152,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.47.attn_norm.weight",
"offset": 22947512320,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.47.ffn_down.weight",
"offset": 22947545088,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.47.ffn_gate.weight",
"offset": 23140220928,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.47.ffn_up.weight",
"offset": 23272341504,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.47.ffn_norm.weight",
"offset": 23404462080,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.47.attn_k.weight",
"offset": 23404494848,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.47.attn_output.weight",
"offset": 23409213440,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.47.attn_q.weight",
"offset": 23446962176,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.47.attn_v.weight",
"offset": 23484710912,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.48.attn_norm.weight",
"offset": 23491592192,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.48.ffn_down.weight",
"offset": 23491624960,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.48.ffn_gate.weight",
"offset": 23623745536,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.48.ffn_up.weight",
"offset": 23755866112,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.48.ffn_norm.weight",
"offset": 23887986688,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.48.attn_k.weight",
"offset": 23888019456,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.48.attn_output.weight",
"offset": 23892738048,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.48.attn_q.weight",
"offset": 23930486784,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.48.attn_v.weight",
"offset": 23968235520,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.49.attn_norm.weight",
"offset": 23974002688,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.49.ffn_down.weight",
"offset": 23974035456,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.49.ffn_gate.weight",
"offset": 24106156032,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.49.ffn_up.weight",
"offset": 24238276608,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.49.ffn_norm.weight",
"offset": 24370397184,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.49.attn_k.weight",
"offset": 24370429952,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.49.attn_output.weight",
"offset": 24375148544,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.49.attn_q.weight",
"offset": 24412897280,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.49.attn_v.weight",
"offset": 24450646016,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.5.attn_norm.weight",
"offset": 24456413184,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.5.ffn_down.weight",
"offset": 24456445952,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.5.ffn_gate.weight",
"offset": 24649121792,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.5.ffn_up.weight",
"offset": 24781242368,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.5.ffn_norm.weight",
"offset": 24913362944,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.5.attn_k.weight",
"offset": 24913395712,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.5.attn_output.weight",
"offset": 24918114304,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.5.attn_q.weight",
"offset": 24955863040,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.5.attn_v.weight",
"offset": 24993611776,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.50.attn_norm.weight",
"offset": 25000493056,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.50.ffn_down.weight",
"offset": 25000525824,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.50.ffn_gate.weight",
"offset": 25132646400,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.50.ffn_up.weight",
"offset": 25264766976,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.50.ffn_norm.weight",
"offset": 25396887552,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.50.attn_k.weight",
"offset": 25396920320,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.50.attn_output.weight",
"offset": 25401638912,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.50.attn_q.weight",
"offset": 25439387648,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.50.attn_v.weight",
"offset": 25477136384,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.51.attn_norm.weight",
"offset": 25482903552,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.51.ffn_down.weight",
"offset": 25482936320,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.51.ffn_gate.weight",
"offset": 25615056896,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.51.ffn_up.weight",
"offset": 25747177472,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.51.ffn_norm.weight",
"offset": 25879298048,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.51.attn_k.weight",
"offset": 25879330816,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.51.attn_output.weight",
"offset": 25884049408,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.51.attn_q.weight",
"offset": 25921798144,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.51.attn_v.weight",
"offset": 25959546880,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.52.attn_norm.weight",
"offset": 25965314048,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.52.ffn_down.weight",
"offset": 25965346816,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.52.ffn_gate.weight",
"offset": 26158022656,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.52.ffn_up.weight",
"offset": 26290143232,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.52.ffn_norm.weight",
"offset": 26422263808,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.52.attn_k.weight",
"offset": 26422296576,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.52.attn_output.weight",
"offset": 26427015168,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.52.attn_q.weight",
"offset": 26464763904,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.52.attn_v.weight",
"offset": 26502512640,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.53.attn_norm.weight",
"offset": 26509393920,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.53.ffn_down.weight",
"offset": 26509426688,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.53.ffn_gate.weight",
"offset": 26641547264,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.53.ffn_up.weight",
"offset": 26773667840,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.53.ffn_norm.weight",
"offset": 26905788416,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.53.attn_k.weight",
"offset": 26905821184,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.53.attn_output.weight",
"offset": 26910539776,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.53.attn_q.weight",
"offset": 26948288512,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.53.attn_v.weight",
"offset": 26986037248,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.54.attn_norm.weight",
"offset": 26991804416,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.54.ffn_down.weight",
"offset": 26991837184,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.54.ffn_gate.weight",
"offset": 27123957760,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.54.ffn_up.weight",
"offset": 27256078336,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.54.ffn_norm.weight",
"offset": 27388198912,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.54.attn_k.weight",
"offset": 27388231680,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.54.attn_output.weight",
"offset": 27392950272,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.54.attn_q.weight",
"offset": 27430699008,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.54.attn_v.weight",
"offset": 27468447744,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.55.attn_norm.weight",
"offset": 27474214912,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.55.ffn_down.weight",
"offset": 27474247680,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.55.ffn_gate.weight",
"offset": 27666923520,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.55.ffn_up.weight",
"offset": 27799044096,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.55.ffn_norm.weight",
"offset": 27931164672,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.55.attn_k.weight",
"offset": 27931197440,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.55.attn_output.weight",
"offset": 27935916032,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.55.attn_q.weight",
"offset": 27973664768,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.55.attn_v.weight",
"offset": 28011413504,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.56.attn_norm.weight",
"offset": 28018294784,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.56.ffn_down.weight",
"offset": 28018327552,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.56.ffn_gate.weight",
"offset": 28150448128,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.56.ffn_up.weight",
"offset": 28282568704,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.56.ffn_norm.weight",
"offset": 28414689280,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.56.attn_k.weight",
"offset": 28414722048,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.56.attn_output.weight",
"offset": 28419440640,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.56.attn_q.weight",
"offset": 28457189376,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.56.attn_v.weight",
"offset": 28494938112,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.57.attn_norm.weight",
"offset": 28500705280,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.57.ffn_down.weight",
"offset": 28500738048,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.57.ffn_gate.weight",
"offset": 28632858624,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.57.ffn_up.weight",
"offset": 28764979200,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.57.ffn_norm.weight",
"offset": 28897099776,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.57.attn_k.weight",
"offset": 28897132544,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.57.attn_output.weight",
"offset": 28901851136,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.57.attn_q.weight",
"offset": 28939599872,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.57.attn_v.weight",
"offset": 28977348608,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.58.attn_norm.weight",
"offset": 28983115776,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.58.ffn_down.weight",
"offset": 28983148544,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.58.ffn_gate.weight",
"offset": 29175824384,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.58.ffn_up.weight",
"offset": 29307944960,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.58.ffn_norm.weight",
"offset": 29440065536,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.58.attn_k.weight",
"offset": 29440098304,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.58.attn_output.weight",
"offset": 29444816896,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.58.attn_q.weight",
"offset": 29482565632,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.58.attn_v.weight",
"offset": 29520314368,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.59.attn_norm.weight",
"offset": 29527195648,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.59.ffn_down.weight",
"offset": 29527228416,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.59.ffn_gate.weight",
"offset": 29659348992,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.59.ffn_up.weight",
"offset": 29791469568,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.59.ffn_norm.weight",
"offset": 29923590144,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.59.attn_k.weight",
"offset": 29923622912,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.59.attn_output.weight",
"offset": 29928341504,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.59.attn_q.weight",
"offset": 29966090240,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.59.attn_v.weight",
"offset": 30003838976,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.6.attn_norm.weight",
"offset": 30009606144,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.6.ffn_down.weight",
"offset": 30009638912,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.6.ffn_gate.weight",
"offset": 30141759488,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.6.ffn_up.weight",
"offset": 30273880064,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.6.ffn_norm.weight",
"offset": 30406000640,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.6.attn_k.weight",
"offset": 30406033408,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.6.attn_output.weight",
"offset": 30410752000,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.6.attn_q.weight",
"offset": 30448500736,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.6.attn_v.weight",
"offset": 30486249472,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.60.attn_norm.weight",
"offset": 30492016640,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.60.ffn_down.weight",
"offset": 30492049408,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.60.ffn_gate.weight",
"offset": 30684725248,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.60.ffn_up.weight",
"offset": 30816845824,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.60.ffn_norm.weight",
"offset": 30948966400,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.60.attn_k.weight",
"offset": 30948999168,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.60.attn_output.weight",
"offset": 30953717760,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.60.attn_q.weight",
"offset": 30991466496,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.60.attn_v.weight",
"offset": 31029215232,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.61.attn_norm.weight",
"offset": 31036096512,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.61.ffn_down.weight",
"offset": 31036129280,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.61.ffn_gate.weight",
"offset": 31168249856,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.61.ffn_up.weight",
"offset": 31300370432,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.61.ffn_norm.weight",
"offset": 31432491008,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.61.attn_k.weight",
"offset": 31432523776,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.61.attn_output.weight",
"offset": 31437242368,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.61.attn_q.weight",
"offset": 31474991104,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.61.attn_v.weight",
"offset": 31512739840,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.62.attn_norm.weight",
"offset": 31518507008,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.62.ffn_down.weight",
"offset": 31518539776,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.62.ffn_gate.weight",
"offset": 31650660352,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.62.ffn_up.weight",
"offset": 31782780928,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.62.ffn_norm.weight",
"offset": 31914901504,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.62.attn_k.weight",
"offset": 31914934272,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.62.attn_output.weight",
"offset": 31919652864,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.62.attn_q.weight",
"offset": 31957401600,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.62.attn_v.weight",
"offset": 31995150336,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.63.attn_norm.weight",
"offset": 32000917504,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.63.ffn_down.weight",
"offset": 32000950272,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.63.ffn_gate.weight",
"offset": 32193626112,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.63.ffn_up.weight",
"offset": 32325746688,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.63.ffn_norm.weight",
"offset": 32457867264,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.63.attn_k.weight",
"offset": 32457900032,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.63.attn_output.weight",
"offset": 32462618624,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.63.attn_q.weight",
"offset": 32500367360,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.63.attn_v.weight",
"offset": 32538116096,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.64.attn_norm.weight",
"offset": 32544997376,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.64.ffn_down.weight",
"offset": 32545030144,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.64.ffn_gate.weight",
"offset": 32677150720,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.64.ffn_up.weight",
"offset": 32809271296,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.64.ffn_norm.weight",
"offset": 32941391872,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.64.attn_k.weight",
"offset": 32941424640,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.64.attn_output.weight",
"offset": 32946143232,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.64.attn_q.weight",
"offset": 32983891968,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.64.attn_v.weight",
"offset": 33021640704,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.65.attn_norm.weight",
"offset": 33027407872,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.65.ffn_down.weight",
"offset": 33027440640,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.65.ffn_gate.weight",
"offset": 33159561216,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.65.ffn_up.weight",
"offset": 33291681792,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.65.ffn_norm.weight",
"offset": 33423802368,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.65.attn_k.weight",
"offset": 33423835136,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.65.attn_output.weight",
"offset": 33428553728,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.65.attn_q.weight",
"offset": 33466302464,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.65.attn_v.weight",
"offset": 33504051200,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.66.attn_norm.weight",
"offset": 33509818368,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.66.ffn_down.weight",
"offset": 33509851136,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.66.ffn_gate.weight",
"offset": 33702526976,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.66.ffn_up.weight",
"offset": 33834647552,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.66.ffn_norm.weight",
"offset": 33966768128,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.66.attn_k.weight",
"offset": 33966800896,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.66.attn_output.weight",
"offset": 33971519488,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.66.attn_q.weight",
"offset": 34009268224,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.66.attn_v.weight",
"offset": 34047016960,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.67.attn_norm.weight",
"offset": 34053898240,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.67.ffn_down.weight",
"offset": 34053931008,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.67.ffn_gate.weight",
"offset": 34186051584,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.67.ffn_up.weight",
"offset": 34318172160,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.67.ffn_norm.weight",
"offset": 34450292736,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.67.attn_k.weight",
"offset": 34450325504,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.67.attn_output.weight",
"offset": 34455044096,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.67.attn_q.weight",
"offset": 34492792832,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.67.attn_v.weight",
"offset": 34530541568,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.68.attn_norm.weight",
"offset": 34536308736,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.68.ffn_down.weight",
"offset": 34536341504,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.68.ffn_gate.weight",
"offset": 34668462080,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.68.ffn_up.weight",
"offset": 34800582656,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.68.ffn_norm.weight",
"offset": 34932703232,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.68.attn_k.weight",
"offset": 34932736000,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.68.attn_output.weight",
"offset": 34937454592,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.68.attn_q.weight",
"offset": 34975203328,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.68.attn_v.weight",
"offset": 35012952064,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.69.attn_norm.weight",
"offset": 35018719232,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.69.ffn_down.weight",
"offset": 35018752000,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.69.ffn_gate.weight",
"offset": 35211427840,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.69.ffn_up.weight",
"offset": 35343548416,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.69.ffn_norm.weight",
"offset": 35475668992,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.69.attn_k.weight",
"offset": 35475701760,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.69.attn_output.weight",
"offset": 35480420352,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.69.attn_q.weight",
"offset": 35518169088,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.69.attn_v.weight",
"offset": 35555917824,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.7.attn_norm.weight",
"offset": 35562799104,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.7.ffn_down.weight",
"offset": 35562831872,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.7.ffn_gate.weight",
"offset": 35694952448,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.7.ffn_up.weight",
"offset": 35827073024,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.7.ffn_norm.weight",
"offset": 35959193600,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.7.attn_k.weight",
"offset": 35959226368,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.7.attn_output.weight",
"offset": 35963944960,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.7.attn_q.weight",
"offset": 36001693696,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.7.attn_v.weight",
"offset": 36039442432,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.70.attn_norm.weight",
"offset": 36045209600,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.70.ffn_down.weight",
"offset": 36045242368,
"shape": [
28672,
8192
],
"size": 32768,
"type": 12
},
{
"name": "blk.70.ffn_gate.weight",
"offset": 36177362944,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.70.ffn_up.weight",
"offset": 36309483520,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.70.ffn_norm.weight",
"offset": 36441604096,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.70.attn_k.weight",
"offset": 36441636864,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.70.attn_output.weight",
"offset": 36446355456,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.70.attn_q.weight",
"offset": 36484104192,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.70.attn_v.weight",
"offset": 36521852928,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 13
},
{
"name": "blk.71.attn_norm.weight",
"offset": 36527620096,
"shape": [
8192
],
"size": 5767168,
"type": 0
},
{
"name": "blk.71.ffn_down.weight",
"offset": 36527652864,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.71.ffn_gate.weight",
"offset": 36720328704,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.71.ffn_up.weight",
"offset": 36852449280,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.71.ffn_norm.weight",
"offset": 36984569856,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.71.attn_k.weight",
"offset": 36984602624,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.71.attn_output.weight",
"offset": 36989321216,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.71.attn_q.weight",
"offset": 37027069952,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.71.attn_v.weight",
"offset": 37064818688,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.72.attn_norm.weight",
"offset": 37071699968,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.72.ffn_down.weight",
"offset": 37071732736,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.72.ffn_gate.weight",
"offset": 37264408576,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.72.ffn_up.weight",
"offset": 37396529152,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.72.ffn_norm.weight",
"offset": 37528649728,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.72.attn_k.weight",
"offset": 37528682496,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.72.attn_output.weight",
"offset": 37533401088,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.72.attn_q.weight",
"offset": 37571149824,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.72.attn_v.weight",
"offset": 37608898560,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.73.attn_norm.weight",
"offset": 37615779840,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.73.ffn_down.weight",
"offset": 37615812608,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.73.ffn_gate.weight",
"offset": 37808488448,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.73.ffn_up.weight",
"offset": 37940609024,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.73.ffn_norm.weight",
"offset": 38072729600,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.73.attn_k.weight",
"offset": 38072762368,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.73.attn_output.weight",
"offset": 38077480960,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.73.attn_q.weight",
"offset": 38115229696,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.73.attn_v.weight",
"offset": 38152978432,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.74.attn_norm.weight",
"offset": 38159859712,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.74.ffn_down.weight",
"offset": 38159892480,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.74.ffn_gate.weight",
"offset": 38352568320,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.74.ffn_up.weight",
"offset": 38484688896,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.74.ffn_norm.weight",
"offset": 38616809472,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.74.attn_k.weight",
"offset": 38616842240,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.74.attn_output.weight",
"offset": 38621560832,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.74.attn_q.weight",
"offset": 38659309568,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.74.attn_v.weight",
"offset": 38697058304,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.75.attn_norm.weight",
"offset": 38703939584,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.75.ffn_down.weight",
"offset": 38703972352,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.75.ffn_gate.weight",
"offset": 38896648192,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.75.ffn_up.weight",
"offset": 39028768768,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.75.ffn_norm.weight",
"offset": 39160889344,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.75.attn_k.weight",
"offset": 39160922112,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.75.attn_output.weight",
"offset": 39165640704,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.75.attn_q.weight",
"offset": 39203389440,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.75.attn_v.weight",
"offset": 39241138176,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.76.attn_norm.weight",
"offset": 39248019456,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.76.ffn_down.weight",
"offset": 39248052224,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.76.ffn_gate.weight",
"offset": 39440728064,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.76.ffn_up.weight",
"offset": 39572848640,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.76.ffn_norm.weight",
"offset": 39704969216,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.76.attn_k.weight",
"offset": 39705001984,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.76.attn_output.weight",
"offset": 39709720576,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.76.attn_q.weight",
"offset": 39747469312,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.76.attn_v.weight",
"offset": 39785218048,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.77.attn_norm.weight",
"offset": 39792099328,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.77.ffn_down.weight",
"offset": 39792132096,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.77.ffn_gate.weight",
"offset": 39984807936,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.77.ffn_up.weight",
"offset": 40116928512,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.77.ffn_norm.weight",
"offset": 40249049088,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.77.attn_k.weight",
"offset": 40249081856,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.77.attn_output.weight",
"offset": 40253800448,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.77.attn_q.weight",
"offset": 40291549184,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.77.attn_v.weight",
"offset": 40329297920,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.78.attn_norm.weight",
"offset": 40336179200,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.78.ffn_down.weight",
"offset": 40336211968,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.78.ffn_gate.weight",
"offset": 40528887808,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.78.ffn_up.weight",
"offset": 40661008384,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.78.ffn_norm.weight",
"offset": 40793128960,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.78.attn_k.weight",
"offset": 40793161728,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.78.attn_output.weight",
"offset": 40797880320,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.78.attn_q.weight",
"offset": 40835629056,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.78.attn_v.weight",
"offset": 40873377792,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.79.attn_norm.weight",
"offset": 40880259072,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.79.ffn_down.weight",
"offset": 40880291840,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.79.ffn_gate.weight",
"offset": 41072967680,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.79.ffn_up.weight",
"offset": 41205088256,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.79.ffn_norm.weight",
"offset": 41337208832,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.79.attn_k.weight",
"offset": 41337241600,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.79.attn_output.weight",
"offset": 41341960192,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.79.attn_q.weight",
"offset": 41379708928,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.79.attn_v.weight",
"offset": 41417457664,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.8.attn_norm.weight",
"offset": 41424338944,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.8.ffn_down.weight",
"offset": 41424371712,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.8.ffn_gate.weight",
"offset": 41617047552,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.8.ffn_up.weight",
"offset": 41749168128,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.8.ffn_norm.weight",
"offset": 41881288704,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.8.attn_k.weight",
"offset": 41881321472,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.8.attn_output.weight",
"offset": 41886040064,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.8.attn_q.weight",
"offset": 41923788800,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.8.attn_v.weight",
"offset": 41961537536,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "blk.9.attn_norm.weight",
"offset": 41968418816,
"shape": [
8192
],
"size": 6881280,
"type": 0
},
{
"name": "blk.9.ffn_down.weight",
"offset": 41968451584,
"shape": [
28672,
8192
],
"size": 32768,
"type": 14
},
{
"name": "blk.9.ffn_gate.weight",
"offset": 42161127424,
"shape": [
8192,
28672
],
"size": 192675840,
"type": 12
},
{
"name": "blk.9.ffn_up.weight",
"offset": 42293248000,
"shape": [
8192,
28672
],
"size": 132120576,
"type": 12
},
{
"name": "blk.9.ffn_norm.weight",
"offset": 42425368576,
"shape": [
8192
],
"size": 132120576,
"type": 0
},
{
"name": "blk.9.attn_k.weight",
"offset": 42425401344,
"shape": [
8192,
1024
],
"size": 32768,
"type": 12
},
{
"name": "blk.9.attn_output.weight",
"offset": 42430119936,
"shape": [
8192,
8192
],
"size": 4718592,
"type": 12
},
{
"name": "blk.9.attn_q.weight",
"offset": 42467868672,
"shape": [
8192,
8192
],
"size": 37748736,
"type": 12
},
{
"name": "blk.9.attn_v.weight",
"offset": 42505617408,
"shape": [
8192,
1024
],
"size": 37748736,
"type": 14
},
{
"name": "output_norm.weight",
"offset": 42512498688,
"shape": [
8192
],
"size": 6881280,
"type": 0
}
],
"version": 3
}